[llvm-branch-commits] [llvm] AMDGPU/GlobalISel: Fix legalizer lowering for G_EXTRACT/INSERT_VECTOR_ELT (PR #210094)
Petar Avramovic via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jul 20 04:37:10 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/210094
>From 2cd71eca8525c0db025a635595c7fe5ef6cff9f8 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Wed, 15 Jul 2026 13:07:29 +0200
Subject: [PATCH] AMDGPU/GlobalISel: Fix legalizer lowering for
G_EXTRACT/INSERT_VECTOR_ELT
Use LLT::integer in bit twiddling lowering for extract/insert vector element.
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 5 +-
.../CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll | 30 +-
.../AMDGPU/GlobalISel/fp-int-conversions.ll | 6 +-
.../AMDGPU/GlobalISel/insertelement.i16.ll | 7192 +++++++----------
.../AMDGPU/GlobalISel/insertelement.i8.ll | 6385 ++++-----------
.../legalize-extract-vector-elt.mir | 120 +-
.../GlobalISel/legalize-insert-vector-elt.mir | 46 +-
.../legalize-llvm.amdgcn.image.dim.a16.ll | 1980 ++---
.../GlobalISel/legalize-shuffle-vector.mir | 16 +-
.../legalize-shuffle-vector.s16.mir | 362 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 11 +-
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 284 +-
llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll | 51 +-
.../AMDGPU/fptrunc.v2f16.no.fast.math.ll | 156 +-
.../AMDGPU/indirect-reg-read-imm-idx.ll | 40 +-
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 1354 +++-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll | 32 +-
llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll | 340 +-
.../test/CodeGen/AMDGPU/load-atomic-global.ll | 847 +-
llvm/test/CodeGen/AMDGPU/load-atomic-local.ll | 804 +-
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll | 310 +-
llvm/test/CodeGen/AMDGPU/mad-mix.ll | 1157 ++-
llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll | 144 +-
.../test/CodeGen/AMDGPU/vector-reduce-fadd.ll | 190 +-
.../test/CodeGen/AMDGPU/vector-reduce-fmul.ll | 190 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll | 20 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 650 +-
.../test/CodeGen/AMDGPU/vector-reduce-smax.ll | 1424 +++-
.../test/CodeGen/AMDGPU/vector-reduce-smin.ll | 1424 +++-
.../test/CodeGen/AMDGPU/vector-reduce-umax.ll | 1414 +++-
.../test/CodeGen/AMDGPU/vector-reduce-umin.ll | 498 +-
32 files changed, 13374 insertions(+), 14116 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index f1d2a808bd68b..2a2f81393d2db 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -208,8 +208,9 @@ static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
const LLT Ty = Query.Types[TypeIdx];
unsigned Size = Ty.getSizeInBits();
assert(Size % 32 == 0);
- return std::pair(
- TypeIdx, LLT::scalarOrVector(ElementCount::getFixed(Size / 32), 32));
+ return std::pair(TypeIdx,
+ LLT::scalarOrVector(ElementCount::getFixed(Size / 32),
+ LLT::integer(32)));
};
}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
index fd5ebda9dbc34..7b87764c5ae1f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
@@ -155,23 +155,22 @@ define <2 x half> @fmul_v2_half_neg_hi(<2 x half> %a, <2 x half> %b) #0 {
; GFX9-LABEL: fmul_v2_half_neg_hi:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
+; GFX9-NEXT: v_xor_b32_e32 v2, 0x80000000, v1
+; GFX9-NEXT: s_mov_b32 s4, 0xffff
+; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v2
+; GFX9-NEXT: v_pk_mul_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: fmul_v2_half_neg_hi:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v2, 0x80000000, v1
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX10-LABEL: fmul_v2_half_neg_hi:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_pk_mul_f16 v0, v0, v1 neg_hi:[0,1]
-; GFX10-NEXT: s_setpc_b64 s[30:31]
%b1 = bitcast <2 x half> %b to float
%b2 = fneg float %b1
%b3 = bitcast float %b2 to <2 x half>
@@ -193,9 +192,12 @@ define <2 x half> @fmul_v2_half_neg_lo1(<2 x half> %a, <2 x half> %b) #0 {
; GFX8-LABEL: fmul_v2_half_neg_lo1:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80000000, v1
+; GFX8-NEXT: v_mul_f16_e32 v1, v0, v1
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: fmul_v2_half_neg_lo1:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll
index 7317d24d3e1be..219563a079697 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fp-int-conversions.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "s_wait" --filter-out "s_delay_alu" --filter-out "endpgm" --filter-out "store" --version 6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck %s --check-prefixes=GCN,FAKE16,PREGFX12,PREGFX12-FAKE16
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,PREGFX12,TRUE16
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,FAKE16,PREGFX12,PREGFX12-FAKE16
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck %s --check-prefixes=GCN,FAKE16,PREGFX12,PREGFX12-FAKE16
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck %s --check-prefixes=GCN,PREGFX12,TRUE16
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck %s --check-prefixes=GCN,FAKE16,PREGFX12,PREGFX12-FAKE16
; RUN: llc -global-isel -mtriple=amdgpu12.00 < %s | FileCheck %s --check-prefixes=GCN,FAKE16,GFX12
define amdgpu_ps void @s_fptoui_f16_to_i16(half inreg %x, ptr addrspace(1) %out) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index 6d6d51a66bfa4..0f963ba85b73e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -1,22 +1,23 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck --check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck --check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-mesa-mesa3d < %s | FileCheck --check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d < %s | FileCheck --check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck --check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck --check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01-mesa-mesa3d < %s | FileCheck --check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck --check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d < %s | FileCheck --check-prefix=GFX11 %s
define amdgpu_ps void @insertelement_s_v2i16_s_s(ptr addrspace(4) inreg %ptr, i16 inreg %val, i32 inreg %idx) {
; GFX9-LABEL: insertelement_s_v2i16_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: s_lshl_b32 s1, s5, 4
-; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s4
+; GFX9-NEXT: s_and_b32 s1, s5, 1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_xor_b32 s2, s2, s0
-; GFX9-NEXT: s_and_b32 s1, s2, s1
-; GFX9-NEXT: s_xor_b32 s0, s1, s0
+; GFX9-NEXT: s_andn2_b32 s0, s0, s1
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v2, s0
; GFX9-NEXT: global_store_dword v[0:1], v2, off
@@ -25,15 +26,14 @@ define amdgpu_ps void @insertelement_s_v2i16_s_s(ptr addrspace(4) inreg %ptr, i1
; GFX8-LABEL: insertelement_s_v2i16_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: s_and_b32 s1, 0xffff, s4
-; GFX8-NEXT: s_lshl_b32 s2, s1, 16
-; GFX8-NEXT: s_or_b32 s1, s1, s2
-; GFX8-NEXT: s_lshl_b32 s2, s5, 4
+; GFX8-NEXT: s_and_b32 s1, s5, 1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 4
+; GFX8-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX8-NEXT: s_lshl_b32 s2, s2, s1
+; GFX8-NEXT: s_lshl_b32 s1, 0xffff, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s1, s1, s0
-; GFX8-NEXT: s_lshl_b32 s2, 0xffff, s2
-; GFX8-NEXT: s_and_b32 s1, s1, s2
-; GFX8-NEXT: s_xor_b32 s0, s1, s0
+; GFX8-NEXT: s_andn2_b32 s0, s0, s1
+; GFX8-NEXT: s_or_b32 s0, s0, s2
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v2, s0
@@ -43,34 +43,34 @@ define amdgpu_ps void @insertelement_s_v2i16_s_s(ptr addrspace(4) inreg %ptr, i1
; GFX7-LABEL: insertelement_s_v2i16_s_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s1, s4, 16
+; GFX7-NEXT: s_and_b32 s1, s5, 1
+; GFX7-NEXT: s_lshl_b32 s1, s1, 4
; GFX7-NEXT: s_and_b32 s2, s4, 0xffff
-; GFX7-NEXT: s_or_b32 s1, s2, s1
-; GFX7-NEXT: s_lshl_b32 s2, s5, 4
+; GFX7-NEXT: s_lshl_b32 s2, s2, s1
+; GFX7-NEXT: s_lshl_b32 s1, 0xffff, s1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_xor_b32 s1, s1, s0
-; GFX7-NEXT: s_lshl_b32 s2, 0xffff, s2
-; GFX7-NEXT: s_and_b32 s1, s1, s2
-; GFX7-NEXT: s_xor_b32 s4, s1, s0
+; GFX7-NEXT: s_andn2_b32 s0, s0, s1
+; GFX7-NEXT: s_or_b32 s2, s0, s2
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v2i16_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX10-NEXT: s_lshl_b32 s1, s5, 4
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s4, s4
-; GFX10-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_and_b32 s1, s5, 1
+; GFX10-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX10-NEXT: s_lshl_b32 s1, s1, 4
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_lshl_b32 s3, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s1, s2, s1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_xor_b32 s2, s2, s0
-; GFX10-NEXT: s_and_b32 s1, s2, s1
-; GFX10-NEXT: s_xor_b32 s0, s1, s0
+; GFX10-NEXT: s_andn2_b32 s0, s0, s3
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
@@ -78,16 +78,16 @@ define amdgpu_ps void @insertelement_s_v2i16_s_s(ptr addrspace(4) inreg %ptr, i1
; GFX11-LABEL: insertelement_s_v2i16_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: s_lshl_b32 s1, s5, 4
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s4
-; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s1, s5, 1
+; GFX11-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: s_lshl_b32 s3, 0xffff, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_xor_b32 s2, s2, s0
+; GFX11-NEXT: s_and_not1_b32 s0, s0, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s2, s1
-; GFX11-NEXT: s_xor_b32 s0, s1, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
@@ -101,30 +101,33 @@ define amdgpu_ps void @insertelement_v_v2i16_s_s(ptr addrspace(1) %ptr, i16 inre
; GFX9-LABEL: insertelement_v_v2i16_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v2, v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s3, 4
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s2
+; GFX9-NEXT: s_and_b32 s0, s3, 1
+; GFX9-NEXT: s_lshl_b32 s0, s0, 4
+; GFX9-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX9-NEXT: s_lshl_b32 s1, s1, s0
; GFX9-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_not_b32 s0, s0
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, s0, v3, v2
+; GFX9-NEXT: v_and_or_b32 v2, v2, s0, v3
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v2i16_s_s:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v2, v[0:1]
-; GFX8-NEXT: s_and_b32 s0, 0xffff, s2
-; GFX8-NEXT: s_lshl_b32 s1, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s1, s3, 4
-; GFX8-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_and_b32 s0, s3, 1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, s0, v2
+; GFX8-NEXT: v_bfi_b32 v2, s0, 0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_or_b32_e32 v2, s1, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -132,44 +135,49 @@ define amdgpu_ps void @insertelement_v_v2i16_s_s(ptr addrspace(1) %ptr, i16 inre
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 16
+; GFX7-NEXT: s_and_b32 s0, s3, 1
; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX7-NEXT: s_lshl_b32 s2, s3, 4
-; GFX7-NEXT: s_or_b32 s0, s1, s0
-; GFX7-NEXT: s_lshl_b32 s1, 0xffff, s2
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, s0, v0
+; GFX7-NEXT: v_bfi_b32 v0, s0, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, s1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v2i16_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v2, v[0:1], off
-; GFX10-NEXT: s_lshl_b32 s0, s3, 4
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s2, s2
-; GFX10-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_and_b32 s0, s3, 1
+; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, s0, s1, v2
+; GFX10-NEXT: v_and_or_b32 v2, v2, s1, s0
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v2i16_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v2, v[0:1], off
-; GFX11-NEXT: s_lshl_b32 s0, s3, 4
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s2, s2
-; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_and_b32 s0, s3, 1
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, s0, s1, v2
+; GFX11-NEXT: v_and_or_b32 v2, v2, s1, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(1 ) %ptr
@@ -182,74 +190,79 @@ define amdgpu_ps void @insertelement_s_v2i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v2i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: s_mov_b32 s1, 0x5040100
-; GFX9-NEXT: v_perm_b32 v2, v0, v0, s1
-; GFX9-NEXT: s_lshl_b32 s1, s4, 4
-; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: s_and_b32 s1, s4, 1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshl_b32 s2, 0xffff, s1
+; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_andn2_b32 s0, s0, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, s1, v3
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v2i16_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v1, 16
-; GFX8-NEXT: s_lshl_b32 s1, s4, 4
-; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: s_and_b32 s1, s4, 1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 4
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX8-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_andn2_b32 s0, s0, s1
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX8-NEXT: v_or_b32_e32 v2, s0, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v2i16_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s1, s4, 4
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: s_and_b32 s1, s4, 1
+; GFX7-NEXT: s_lshl_b32 s1, s1, 4
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s1, v0
; GFX7-NEXT: s_lshl_b32 s1, 0xffff, s1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, v0, s0
+; GFX7-NEXT: s_andn2_b32 s0, s0, s1
+; GFX7-NEXT: v_or_b32_e32 v0, s0, v0
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v2i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX10-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
-; GFX10-NEXT: s_lshl_b32 s1, s4, 4
+; GFX10-NEXT: s_and_b32 s1, s4, 1
+; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX10-NEXT: s_lshl_b32 s1, s1, 4
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX10-NEXT: s_lshl_b32 s2, 0xffff, s1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, s1, v2, s0
+; GFX10-NEXT: s_andn2_b32 s0, s0, s2
+; GFX10-NEXT: v_lshl_or_b32 v2, v2, s1, s0
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX11-NEXT: s_lshl_b32 s1, s4, 4
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_and_b32 s1, s4, 1
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xffff, v0
+; GFX11-NEXT: s_lshl_b32 s1, s1, 4
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, s1, v2, s0
+; GFX11-NEXT: s_and_not1_b32 s0, s0, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshl_or_b32 v2, v2, s1, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(4) %ptr
@@ -262,75 +275,85 @@ define amdgpu_ps void @insertelement_s_v2i16_s_v(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v2i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX9-NEXT: s_mov_b32 s1, 0xffff
+; GFX9-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: v_lshlrev_b32_e64 v2, v0, s1
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s4, s4
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_not_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v2, v3, s0
+; GFX9-NEXT: v_and_or_b32 v2, s0, v3, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v2i16_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: s_and_b32 s2, 0xffff, s4
-; GFX8-NEXT: s_mov_b32 s1, 0xffff
-; GFX8-NEXT: s_lshl_b32 s3, s2, 16
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX8-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v0, s1
+; GFX8-NEXT: v_bfi_b32 v3, v0, 0, s0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_bfi_b32 v2, v3, s2, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v2i16_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s1, s4, 16
-; GFX7-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX7-NEXT: s_or_b32 s1, s2, s1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX7-NEXT: v_lshl_b32_e32 v1, s1, v0
; GFX7-NEXT: v_lshl_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_bfi_b32 v0, v0, s1, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, s0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v2i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT: s_and_b32 s1, s4, 0xffff
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s4, s4
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, s1
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_not_b32_e32 v3, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v2, s1, s0
+; GFX10-NEXT: v_and_or_b32 v2, s0, v3, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v2i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 4, v0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s4, s4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX11-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v0, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, s1
; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v3, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v2, s1, s0
+; GFX11-NEXT: v_and_or_b32 v2, s0, v3, v2
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(4) %ptr
@@ -343,75 +366,82 @@ define amdgpu_ps void @insertelement_s_v2i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v2i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: s_mov_b32 s1, 0x5040100
-; GFX9-NEXT: v_perm_b32 v2, v0, v0, s1
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v1
-; GFX9-NEXT: s_mov_b32 s1, 0xffff
-; GFX9-NEXT: v_lshlrev_b32_e64 v3, v0, s1
+; GFX9-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v1, v2
+; GFX9-NEXT: v_not_b32_e32 v2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX9-NEXT: v_and_or_b32 v2, s0, v2, v3
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v2i16_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v2, 16
-; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: s_mov_b32 s1, 0xffff
-; GFX8-NEXT: v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v1
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v0, s1
+; GFX8-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v1, v2
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_bfi_b32 v2, v0, 0, s0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v2i16_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, v0, s0
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, s0
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v2i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v1, 1, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX10-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v1, 0xffff
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_not_b32_e32 v2, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX10-NEXT: v_and_or_b32 v2, s0, v2, v3
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v2i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX11-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v3.h, v0.l
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_lshlrev_b32 v3, v1, v0
; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: v_not_b32_e32 v2, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v2, v3, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v2, s0, v2, v3
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(4) %ptr
@@ -424,30 +454,34 @@ define amdgpu_ps void @insertelement_v_v2i16_s_v(ptr addrspace(1) %ptr, i16 inre
; GFX9-LABEL: insertelement_v_v2i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v3, v[0:1], off
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v2
-; GFX9-NEXT: s_mov_b32 s0, 0xffff
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
; GFX9-NEXT: v_lshlrev_b32_e64 v2, v0, s0
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_not_b32_e32 v4, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX9-NEXT: v_and_or_b32 v2, v3, v4, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v2i16_s_v:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
-; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
-; GFX8-NEXT: s_mov_b32 s0, 0xffff
-; GFX8-NEXT: s_lshl_b32 s2, s1, 16
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v2
-; GFX8-NEXT: s_or_b32 s1, s1, s2
-; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX8-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v2, v1
+; GFX8-NEXT: v_lshlrev_b32_e64 v3, v2, s0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfi_b32 v2, v1, 0, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v2, s1, v3
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -455,44 +489,52 @@ define amdgpu_ps void @insertelement_v_v2i16_s_v(ptr addrspace(1) %ptr, i16 inre
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 16
-; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v2
-; GFX7-NEXT: s_or_b32 s0, s1, s0
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_lshl_b32_e32 v2, s0, v1
; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, s0, v0
+; GFX7-NEXT: v_bfi_b32 v0, v1, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v2i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v2
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX10-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, s0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_not_b32_e32 v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX10-NEXT: v_and_or_b32 v2, v3, v4, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v2i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 4, v2
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s2, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v0, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, s0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX11-NEXT: v_and_or_b32 v2, v3, v4, v2
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(1) %ptr
@@ -505,29 +547,31 @@ define amdgpu_ps void @insertelement_v_v2i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX9-LABEL: insertelement_v_v2i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v3, v[0:1], off
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX9-NEXT: s_lshl_b32 s0, s2, 4
+; GFX9-NEXT: s_and_b32 s0, s2, 1
+; GFX9-NEXT: s_lshl_b32 s0, s0, 4
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX9-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX9-NEXT: s_not_b32 s0, s0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, s0, v2, v3
+; GFX9-NEXT: v_and_or_b32 v2, v3, s0, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v2i16_v_s:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, 16
-; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: s_lshl_b32 s0, s2, 4
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_and_b32 s0, s2, 1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, s0, v2, v3
+; GFX8-NEXT: v_bfi_b32 v3, s0, 0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -535,44 +579,50 @@ define amdgpu_ps void @insertelement_v_v2i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_lshl_b32 s0, s2, 4
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: s_and_b32 s0, s2, 1
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, s0, v1
; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, s0, v1, v0
+; GFX7-NEXT: v_bfi_b32 v0, s0, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v2i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-NEXT: v_perm_b32 v2, v2, v2, 0x5040100
-; GFX10-NEXT: s_lshl_b32 s0, s2, 4
+; GFX10-NEXT: s_and_b32 s0, s2, 1
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_not_b32 s0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, s0, v2, v3
+; GFX10-NEXT: v_and_or_b32 v2, v3, s0, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v2i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-NEXT: s_lshl_b32 s0, s2, 4
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xffff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 1
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, s0, v0
; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, s0, v4, v3
+; GFX11-NEXT: v_and_or_b32 v2, v3, s0, v2
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(1) %ptr
@@ -585,31 +635,32 @@ define amdgpu_ps void @insertelement_v_v2i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX9-LABEL: insertelement_v_v2i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v3
-; GFX9-NEXT: s_mov_b32 s0, 0xffff
-; GFX9-NEXT: v_lshlrev_b32_e64 v3, v0, s0
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_not_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, v2, v4
+; GFX9-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v2i16_v_v:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, 16
-; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: s_mov_b32 s0, 0xffff
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v3
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v0, s0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfi_b32 v3, v1, 0, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -617,45 +668,51 @@ define amdgpu_ps void @insertelement_v_v2i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 4, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v2, v1, v0
+; GFX7-NEXT: v_bfi_b32 v0, v1, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v2i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v3
-; GFX10-NEXT: v_perm_b32 v2, v2, v2, 0x5040100
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_not_b32_e32 v3, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v3, v2, v4
+; GFX10-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v2i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 4, v3
-; GFX11-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v5.h, v2.l
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v0, 0xffff
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v3
+; GFX11-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_lshlrev_b32 v3, v0, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: v_not_b32_e32 v2, v2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v3, v5, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v2, v4, v2, v3
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <2 x i16>, ptr addrspace(1) %ptr
@@ -725,34 +782,43 @@ define amdgpu_ps void @insertelement_v_v4i16_s_s(ptr addrspace(1) %ptr, i16 inre
; GFX9-LABEL: insertelement_v_v4i16_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s3, 4
-; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s2
-; GFX9-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
+; GFX9-NEXT: s_and_b32 s1, s3, 1
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshr_b32 s0, s3, 1
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v4, v3, s1, v2
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX9-NEXT: v_bfi_b32 v0, s0, v5, v0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i16_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX8-NEXT: s_lshl_b32 s0, s3, 4
-; GFX8-NEXT: s_lshl_b32 s3, s2, 16
-; GFX8-NEXT: s_or_b32 s2, s2, s3
-; GFX8-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: s_and_b32 s0, s3, 1
+; GFX8-NEXT: s_lshr_b32 m0, s3, 1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX8-NEXT: v_bfi_b32 v0, s0, v5, v0
+; GFX8-NEXT: v_movrels_b32_e32 v2, v0
+; GFX8-NEXT: v_bfi_b32 v2, s0, 0, v2
+; GFX8-NEXT: v_or_b32_e32 v4, s1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_movreld_b32_e32 v0, v4
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -760,48 +826,60 @@ define amdgpu_ps void @insertelement_v_v4i16_s_s(ptr addrspace(1) %ptr, i16 inre
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 16
+; GFX7-NEXT: s_and_b32 s0, s3, 1
+; GFX7-NEXT: s_lshr_b32 m0, s3, 1
; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX7-NEXT: s_lshl_b32 s2, s3, 4
-; GFX7-NEXT: s_or_b32 s3, s1, s0
-; GFX7-NEXT: s_lshl_b64 s[0:1], 0xffff, s2
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX7-NEXT: v_bfi_b32 v0, s0, v3, v0
+; GFX7-NEXT: v_movrels_b32_e32 v2, v0
+; GFX7-NEXT: v_bfi_b32 v2, s0, 0, v2
+; GFX7-NEXT: v_or_b32_e32 v2, s1, v2
+; GFX7-NEXT: v_movreld_b32_e32 v0, v2
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i16_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: s_lshl_b32 s0, s3, 4
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s2, s2
-; GFX10-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: s_and_b32 s0, s3, 1
+; GFX10-NEXT: s_lshr_b32 m0, s3, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, s1, s2, v1
-; GFX10-NEXT: v_bfi_b32 v0, s0, s2, v0
+; GFX10-NEXT: v_movrels_b32_e32 v2, v0
+; GFX10-NEXT: v_and_or_b32 v4, v2, s1, s0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v4
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v4i16_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_lshl_b32 s0, s3, 4
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s2
-; GFX11-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: s_and_b32 s0, s3, 1
+; GFX11-NEXT: s_lshr_b32 m0, s3, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, s1, s2, v1
-; GFX11-NEXT: v_bfi_b32 v0, s0, s2, v0
+; GFX11-NEXT: v_movrels_b32_e32 v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v4, v2, s1, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v4
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(1 ) %ptr
@@ -814,83 +892,108 @@ define amdgpu_ps void @insertelement_s_v4i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v4i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: v_perm_b32 v0, v0, v0, s2
-; GFX9-NEXT: s_lshl_b32 s2, s4, 4
-; GFX9-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
+; GFX9-NEXT: s_lshr_b32 s2, s4, 1
+; GFX9-NEXT: s_and_b32 s4, s4, 1
+; GFX9-NEXT: s_mov_b32 m0, s2
+; GFX9-NEXT: s_lshl_b32 s4, s4, 4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_movrels_b32 s3, s0
+; GFX9-NEXT: s_lshl_b32 s5, 0xffff, s4
+; GFX9-NEXT: s_andn2_b32 s3, s3, s5
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_lshl_or_b32 v4, v0, s4, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_bfi_b32 v1, s3, v0, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_bfi_b32 v0, s2, v0, v4
+; GFX9-NEXT: s_set_gpr_idx_on s2, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v4i16_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v1, 16
-; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: s_lshl_b32 s2, s4, 4
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
+; GFX8-NEXT: s_and_b32 s3, s4, 1
+; GFX8-NEXT: s_lshr_b32 m0, s4, 1
+; GFX8-NEXT: s_lshl_b32 s3, s3, 4
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_movrels_b32 s2, s0
+; GFX8-NEXT: s_lshl_b32 s3, 0xffff, s3
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: s_andn2_b32 s2, s2, s3
+; GFX8-NEXT: v_or_b32_e32 v4, s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_bfi_b32 v1, s3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_bfi_b32 v0, s2, v0, v4
+; GFX8-NEXT: v_movreld_b32_e32 v0, v4
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v4i16_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: s_and_b32 s3, s4, 1
+; GFX7-NEXT: s_lshr_b32 m0, s4, 1
+; GFX7-NEXT: s_lshl_b32 s3, s3, 4
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_lshl_b32 s2, s4, 4
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_movrels_b32 s2, s0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX7-NEXT: s_lshl_b32 s3, 0xffff, s3
+; GFX7-NEXT: s_andn2_b32 s2, s2, s3
+; GFX7-NEXT: v_or_b32_e32 v2, s2, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_bfi_b32 v1, s3, v0, v1
-; GFX7-NEXT: v_bfi_b32 v0, s2, v0, v2
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_movreld_b32_e32 v0, v2
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v4i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_perm_b32 v2, v0, v0, 0x5040100
-; GFX10-NEXT: s_lshl_b32 s2, s4, 4
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_and_b32 s2, s4, 1
+; GFX10-NEXT: s_lshr_b32 m0, s4, 1
+; GFX10-NEXT: s_lshl_b32 s2, s2, 4
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v3, s3, v2, s1
-; GFX10-NEXT: v_bfi_b32 v2, s2, v2, s0
-; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-NEXT: s_movrels_b32 s4, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_andn2_b32 s3, s4, s3
+; GFX10-NEXT: v_lshl_or_b32 v4, v0, s2, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v4
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v2.h, v0.l
-; GFX11-NEXT: s_lshl_b32 s2, s4, 4
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_lshl_b64 s[2:3], 0xffff, s2
+; GFX11-NEXT: s_and_b32 s2, s4, 1
+; GFX11-NEXT: s_lshr_b32 m0, s4, 1
+; GFX11-NEXT: s_lshl_b32 s2, s2, 4
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v3, s3, v2, s1
-; GFX11-NEXT: v_bfi_b32 v2, s2, v2, s0
-; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX11-NEXT: s_movrels_b32 s4, s0
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_and_not1_b32 s3, s4, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v4, v0, s2, s3
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v4
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <4 x i16> %vec, i16 %val, i32 %idx
@@ -902,85 +1005,124 @@ define amdgpu_ps void @insertelement_s_v4i16_s_v(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v4i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0xffff
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, s[2:3]
-; GFX9-NEXT: s_pack_ll_b32_b16 s2, s4, s4
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_bfi_b32 v1, v1, s2, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX9-NEXT: v_lshlrev_b32_e64 v5, v0, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_and_or_b32 v5, v4, v0, v5
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v1, v5, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_bfi_b32 v0, v0, s2, v4
-; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v4i16_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: s_and_b32 s2, 0xffff, s4
-; GFX8-NEXT: s_lshl_b32 s3, s2, 16
-; GFX8-NEXT: s_or_b32 s4, s2, s3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX8-NEXT: s_mov_b64 s[2:3], 0xffff
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v0, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_bfi_b32 v1, v1, s4, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX8-NEXT: v_mov_b32_e32 v6, 0xffff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v0, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v6
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v4
+; GFX8-NEXT: v_or_b32_e32 v5, v0, v5
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v1, v5, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_bfi_b32 v0, v0, s4, v4
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v4i16_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s2, s4, 16
-; GFX7-NEXT: s_and_b32 s3, s4, 0xffff
-; GFX7-NEXT: s_or_b32 s4, s3, s2
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 1, v0
+; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX7-NEXT: s_mov_b64 s[2:3], 0xffff
-; GFX7-NEXT: v_lshl_b64 v[0:1], s[2:3], v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s1
-; GFX7-NEXT: v_bfi_b32 v1, v1, s4, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_bfi_b32 v0, v0, s4, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX7-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX7-NEXT: v_lshl_b32_e32 v5, s0, v0
+; GFX7-NEXT: v_lshl_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, v4
+; GFX7-NEXT: v_or_b32_e32 v4, v0, v5
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v1, v4, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v4i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s4, s4
+; GFX10-NEXT: v_and_b32_e32 v1, 1, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 1, v0
+; GFX10-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 4, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v2, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v2, s2
+; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v5, v5, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], v0, 0xffff
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, v1, s2, s1
-; GFX10-NEXT: v_bfi_b32 v0, v0, s2, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v0, 4, v0
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s4, s4
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, 0xffff
+; GFX11-NEXT: v_and_b32_e32 v1, 1, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 1, v0
+; GFX11-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, v1, s2, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_bfi_b32 v0, v0, s2, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 4, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v2, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v2, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v5, v5, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
@@ -993,82 +1135,122 @@ define amdgpu_ps void @insertelement_s_v4i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v4i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 1, v1
+; GFX9-NEXT: v_and_b32_e32 v1, 1, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX9-NEXT: s_mov_b64 s[2:3], 0xffff
-; GFX9-NEXT: s_mov_b32 s4, 0x5040100
-; GFX9-NEXT: v_lshlrev_b64 v[1:2], v1, s[2:3]
-; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, v1, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v3, vcc
+; GFX9-NEXT: v_not_b32_e32 v1, v1
+; GFX9-NEXT: v_and_or_b32 v1, v5, v1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX9-NEXT: global_store_dwordx2 v[3:4], v[1:2], off
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v4i16_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 1, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX8-NEXT: s_mov_b64 s[2:3], 0xffff
-; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: v_lshlrev_b64 v[1:2], v1, s[2:3]
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v6, 0xffff
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v2, v3, vcc
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v1, v6
+; GFX8-NEXT: v_bfi_b32 v1, v1, 0, v5
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[1:2]
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v4i16_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 4, v1
-; GFX7-NEXT: s_mov_b64 s[2:3], 0xffff
-; GFX7-NEXT: v_lshl_b64 v[0:1], s[2:3], v0
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 1, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, v1, v2, s1
-; GFX7-NEXT: v_bfi_b32 v0, v0, v2, s0
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v2, v3, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v5
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v4i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 4, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v0, 0x5040100
+; GFX10-NEXT: v_and_b32_e32 v2, 1, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 1, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 4, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v3, v4
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v4, v3, v0
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_lshlrev_b64 v[1:2], v1, 0xffff
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX10-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX10-NEXT: global_store_dwordx2 v[3:4], v[1:2], off
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v6, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_lshlrev_b32 v1, 4, v1
-; GFX11-NEXT: v_mov_b16_e32 v5.l, v0.l
-; GFX11-NEXT: v_mov_b16_e32 v5.h, v0.l
-; GFX11-NEXT: v_mov_b32_e32 v3, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[1:2], v1, 0xffff
+; GFX11-NEXT: v_and_b32_e32 v2, 1, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v2, v5, s1
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 4, v2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v2, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, v2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v3, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_bfi_b32 v1, v1, v5, s0
-; GFX11-NEXT: global_store_b64 v[3:4], v[1:2], off
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <4 x i16> %vec, i16 %val, i32 %idx
@@ -1080,32 +1262,46 @@ define amdgpu_ps void @insertelement_v_v4i16_s_v(ptr addrspace(1) %ptr, i16 inre
; GFX9-LABEL: insertelement_v_v4i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b64 s[0:1], 0xffff
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 1, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT: v_mov_b32_e32 v5, 0xffff
+; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 4, v2
-; GFX9-NEXT: v_lshlrev_b64 v[5:6], v2, s[0:1]
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s2
+; GFX9-NEXT: v_lshlrev_b32_e64 v7, v2, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, v2, v5
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX9-NEXT: v_not_b32_e32 v2, v2
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v6
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, v6, s0, v1
-; GFX9-NEXT: v_bfi_b32 v0, v5, s0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX9-NEXT: v_and_or_b32 v2, v5, v2, v7
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX9-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i16_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[0:1], 0xffff
-; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 1, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, 0xffff
+; GFX8-NEXT: s_and_b32 s0, s2, 0xffff
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 4, v2
-; GFX8-NEXT: s_lshl_b32 s3, s2, 16
-; GFX8-NEXT: v_lshlrev_b64 v[5:6], v2, s[0:1]
-; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX8-NEXT: v_lshlrev_b32_e64 v7, v2, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v2, v5
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v6
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, v6, s2, v1
-; GFX8-NEXT: v_bfi_b32 v0, v5, s2, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX8-NEXT: v_bfi_b32 v2, v2, 0, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v7
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -1113,50 +1309,71 @@ define amdgpu_ps void @insertelement_v_v4i16_s_v(ptr addrspace(1) %ptr, i16 inre
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[0:1], 0xffff
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 1, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xffff
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 4, v2
-; GFX7-NEXT: s_lshl_b32 s3, s2, 16
-; GFX7-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX7-NEXT: v_lshl_b64 v[2:3], s[0:1], v2
-; GFX7-NEXT: s_or_b32 s2, s2, s3
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, s0, v2
+; GFX7-NEXT: v_lshl_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, v3, s2, v1
-; GFX7-NEXT: v_bfi_b32 v0, v2, s2, v0
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX7-NEXT: v_bfi_b32 v2, v2, 0, v5
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 4, v2
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v2, 0xffff
+; GFX10-NEXT: v_and_b32_e32 v3, 1, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 1, v2
+; GFX10-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v3, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX10-NEXT: v_not_b32_e32 v3, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, v3, s0, v1
-; GFX10-NEXT: v_bfi_b32 v0, v2, s0, v0
-; GFX10-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v4i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_lshlrev_b32 v2, 4, v2
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s2, s2
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v2, 0xffff
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 1, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX11-NEXT: v_and_b32_e32 v3, 1, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v3, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX11-NEXT: v_not_b32_e32 v3, v4
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, v3, s0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_bfi_b32 v0, v2, s0, v0
-; GFX11-NEXT: global_store_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(1) %ptr
%insert = insertelement <4 x i16> %vec, i16 %val, i32 %idx
@@ -1168,79 +1385,102 @@ define amdgpu_ps void @insertelement_v_v4i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX9-LABEL: insertelement_v_v4i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX9-NEXT: s_lshl_b32 s0, s2, 4
-; GFX9-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_and_b32 s1, s2, 1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshr_b32 s0, s2, 1
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, s1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX9-NEXT: v_bfi_b32 v0, s0, v2, v0
-; GFX9-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v4, v3, s1, v2
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i16_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v5, 16
-; GFX8-NEXT: v_lshlrev_b32_sdwa v5, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: s_lshl_b32 s0, s2, 4
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-NEXT: s_and_b32 s0, s2, 1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshr_b32 m0, s2, 1
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX8-NEXT: v_bfi_b32 v0, s0, v2, v0
-; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-NEXT: v_movrels_b32_e32 v3, v0
+; GFX8-NEXT: v_bfi_b32 v3, s0, 0, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v3, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_movreld_b32_e32 v0, v4
+; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v4i16_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 1
+; GFX7-NEXT: s_lshr_b32 m0, s2, 1
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_lshl_b32 s0, s2, 4
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, s0, v2
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX7-NEXT: v_bfi_b32 v0, s0, v2, v0
+; GFX7-NEXT: v_movrels_b32_e32 v3, v0
+; GFX7-NEXT: v_bfi_b32 v3, s0, 0, v3
+; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_movreld_b32_e32 v0, v2
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_perm_b32 v4, v2, v2, 0x5040100
-; GFX10-NEXT: s_lshl_b32 s0, s2, 4
+; GFX10-NEXT: s_and_b32 s0, s2, 1
+; GFX10-NEXT: s_lshr_b32 m0, s2, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_not_b32 s0, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_movrels_b32_e32 v3, v0
+; GFX10-NEXT: v_and_or_b32 v4, v3, s0, v2
; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX10-NEXT: v_bfi_b32 v0, s0, v4, v0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v4
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v4i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v4.h, v2.l
-; GFX11-NEXT: s_lshl_b32 s0, s2, 4
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 1
+; GFX11-NEXT: s_lshr_b32 m0, s2, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, s0, v2
+; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX11-NEXT: v_bfi_b32 v0, s0, v4, v0
+; GFX11-NEXT: v_movrels_b32_e32 v3, v0
+; GFX11-NEXT: v_and_or_b32 v4, v3, s0, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_movreld_b32_e32 v0, v4
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(1) %ptr
@@ -1253,85 +1493,116 @@ define amdgpu_ps void @insertelement_v_v4i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX9-LABEL: insertelement_v_v4i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b64 s[0:1], 0xffff
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 1, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 4, v3
-; GFX9-NEXT: s_mov_b32 s2, 0x5040100
-; GFX9-NEXT: v_lshlrev_b64 v[6:7], v3, s[0:1]
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, v3, v6
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
+; GFX9-NEXT: v_not_b32_e32 v3, v3
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v7
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, v7, v2, v1
-; GFX9-NEXT: v_bfi_b32 v0, v6, v2, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
+; GFX9-NEXT: v_and_or_b32 v2, v6, v3, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i16_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v6, 16
-; GFX8-NEXT: s_mov_b64 s[0:1], 0xffff
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 1, v3
+; GFX8-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_mov_b32_e32 v6, 0xffff
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 4, v3
-; GFX8-NEXT: v_lshlrev_b32_sdwa v8, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: v_lshlrev_b64 v[6:7], v3, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, v3, v6
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v7
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, v7, v2, v1
-; GFX8-NEXT: v_bfi_b32 v0, v6, v2, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
+; GFX8-NEXT: v_bfi_b32 v3, v3, 0, v6
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX8-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v4i16_v_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: s_mov_b64 s[0:1], 0xffff
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 1, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 1, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 4, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
-; GFX7-NEXT: v_lshl_b64 v[2:3], s[0:1], v3
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xffff, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, v3, v4, v1
-; GFX7-NEXT: v_bfi_b32 v0, v2, v4, v0
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX7-NEXT: v_bfi_b32 v3, v3, 0, v5
+; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 4, v3
-; GFX10-NEXT: v_perm_b32 v2, v2, v2, 0x5040100
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_mov_b32_e32 v6, 0
-; GFX10-NEXT: v_lshlrev_b64 v[3:4], v3, 0xffff
+; GFX10-NEXT: v_and_b32_e32 v4, 1, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 1, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 4, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v6
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v3, v5
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, v4, v2, v1
-; GFX10-NEXT: v_bfi_b32 v0, v3, v2, v0
-; GFX10-NEXT: global_store_dwordx2 v[5:6], v[0:1], off
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v4i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_lshlrev_b32 v3, 4, v3
-; GFX11-NEXT: v_mov_b16_e32 v7.l, v2.l
-; GFX11-NEXT: v_mov_b16_e32 v7.h, v2.l
-; GFX11-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b64 v[3:4], v3, 0xffff
+; GFX11-NEXT: v_and_b32_e32 v4, 1, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 4, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v3, v5
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, v4, v7, v1
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_bfi_b32 v0, v3, v7, v0
-; GFX11-NEXT: global_store_b64 v[5:6], v[0:1], off
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i16>, ptr addrspace(1) %ptr
%insert = insertelement <4 x i16> %vec, i16 %val, i32 %idx
@@ -1343,33 +1614,19 @@ define amdgpu_ps void @insertelement_s_v8i16_s_s(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v8i16_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: s_cmp_eq_u32 s5, 6
+; GFX9-NEXT: s_lshr_b32 m0, s5, 1
+; GFX9-NEXT: s_and_b32 s5, s5, 1
+; GFX9-NEXT: s_lshl_b32 s5, s5, 4
+; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_movrels_b32 s6, s0
+; GFX9-NEXT: s_lshl_b32 s4, s4, s5
+; GFX9-NEXT: s_lshl_b32 s5, 0xffff, s5
+; GFX9-NEXT: s_andn2_b32 s5, s6, s5
+; GFX9-NEXT: s_or_b32 s4, s5, s4
+; GFX9-NEXT: s_movreld_b32 s0, s4
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cselect_b32 s6, s4, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 7
-; GFX9-NEXT: s_cselect_b32 s3, s4, s3
-; GFX9-NEXT: s_cmp_eq_u32 s5, 4
-; GFX9-NEXT: s_pack_ll_b32_b16 s3, s6, s3
-; GFX9-NEXT: s_cselect_b32 s6, s4, s2
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 5
-; GFX9-NEXT: s_cselect_b32 s2, s4, s2
-; GFX9-NEXT: s_cmp_eq_u32 s5, 2
-; GFX9-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX9-NEXT: s_cselect_b32 s6, s4, s1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 3
-; GFX9-NEXT: s_cselect_b32 s1, s4, s1
-; GFX9-NEXT: s_cmp_eq_u32 s5, 0
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX9-NEXT: s_cselect_b32 s6, s4, s0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 1
-; GFX9-NEXT: s_cselect_b32 s0, s4, s0
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s6, s0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -1380,41 +1637,19 @@ define amdgpu_ps void @insertelement_s_v8i16_s_s(ptr addrspace(4) inreg %ptr, i1
; GFX8-LABEL: insertelement_s_v8i16_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: s_cmp_eq_u32 s5, 6
+; GFX8-NEXT: s_lshr_b32 m0, s5, 1
+; GFX8-NEXT: s_and_b32 s5, s5, 1
+; GFX8-NEXT: s_lshl_b32 s5, s5, 4
+; GFX8-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_movrels_b32 s6, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, s5
+; GFX8-NEXT: s_lshl_b32 s5, 0xffff, s5
+; GFX8-NEXT: s_andn2_b32 s5, s6, s5
+; GFX8-NEXT: s_or_b32 s4, s5, s4
+; GFX8-NEXT: s_movreld_b32 s0, s4
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_cselect_b32 s6, s4, s3
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 7
-; GFX8-NEXT: s_cselect_b32 s3, s4, s3
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_or_b32 s3, s6, s3
-; GFX8-NEXT: s_cmp_eq_u32 s5, 4
-; GFX8-NEXT: s_cselect_b32 s6, s4, s2
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 5
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_or_b32 s2, s6, s2
-; GFX8-NEXT: s_cmp_eq_u32 s5, 2
-; GFX8-NEXT: s_cselect_b32 s6, s4, s1
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 3
-; GFX8-NEXT: s_cselect_b32 s1, s4, s1
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_or_b32 s1, s6, s1
-; GFX8-NEXT: s_cmp_eq_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s6, s4, s0
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 1
-; GFX8-NEXT: s_cselect_b32 s0, s4, s0
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s6, s0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s2
@@ -1425,119 +1660,69 @@ define amdgpu_ps void @insertelement_s_v8i16_s_s(ptr addrspace(4) inreg %ptr, i1
; GFX7-LABEL: insertelement_s_v8i16_s_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX7-NEXT: s_cmp_eq_u32 s5, 6
+; GFX7-NEXT: s_lshr_b32 m0, s5, 1
+; GFX7-NEXT: s_and_b32 s5, s5, 1
+; GFX7-NEXT: s_lshl_b32 s5, s5, 4
+; GFX7-NEXT: s_and_b32 s4, s4, 0xffff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_cselect_b32 s6, s4, s3
-; GFX7-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX7-NEXT: s_lshr_b32 s3, s3, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 7
-; GFX7-NEXT: s_cselect_b32 s3, s4, s3
-; GFX7-NEXT: s_lshl_b32 s3, s3, 16
-; GFX7-NEXT: s_or_b32 s3, s6, s3
-; GFX7-NEXT: s_cmp_eq_u32 s5, 4
-; GFX7-NEXT: s_cselect_b32 s6, s4, s2
-; GFX7-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX7-NEXT: s_lshr_b32 s2, s2, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 5
-; GFX7-NEXT: s_cselect_b32 s2, s4, s2
-; GFX7-NEXT: s_lshl_b32 s2, s2, 16
-; GFX7-NEXT: s_or_b32 s2, s6, s2
-; GFX7-NEXT: s_cmp_eq_u32 s5, 2
-; GFX7-NEXT: s_cselect_b32 s6, s4, s1
-; GFX7-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 3
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 16
-; GFX7-NEXT: s_or_b32 s1, s6, s1
-; GFX7-NEXT: s_cmp_eq_u32 s5, 0
-; GFX7-NEXT: s_cselect_b32 s6, s4, s0
-; GFX7-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 1
-; GFX7-NEXT: s_cselect_b32 s0, s4, s0
-; GFX7-NEXT: s_lshl_b32 s0, s0, 16
-; GFX7-NEXT: s_or_b32 s0, s6, s0
+; GFX7-NEXT: s_movrels_b32 s6, s0
+; GFX7-NEXT: s_lshl_b32 s4, s4, s5
+; GFX7-NEXT: s_lshl_b32 s5, 0xffff, s5
+; GFX7-NEXT: s_andn2_b32 s5, s6, s5
+; GFX7-NEXT: s_or_b32 s4, s5, s4
+; GFX7-NEXT: s_movreld_b32 s0, s4
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i16_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: s_cmp_eq_u32 s5, 6
+; GFX10-NEXT: s_lshr_b32 m0, s5, 1
+; GFX10-NEXT: s_and_b32 s5, s5, 1
+; GFX10-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX10-NEXT: s_lshl_b32 s5, s5, 4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_lshl_b32 s7, 0xffff, s5
+; GFX10-NEXT: s_lshl_b32 s4, s4, s5
; GFX10-NEXT: v_mov_b32_e32 v5, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cselect_b32 s6, s4, s3
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 7
-; GFX10-NEXT: s_cselect_b32 s3, s4, s3
-; GFX10-NEXT: s_cmp_eq_u32 s5, 4
-; GFX10-NEXT: s_pack_ll_b32_b16 s3, s6, s3
-; GFX10-NEXT: s_cselect_b32 s7, s4, s2
-; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 5
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: s_cselect_b32 s2, s4, s2
-; GFX10-NEXT: s_cmp_eq_u32 s5, 2
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s7, s2
-; GFX10-NEXT: s_cselect_b32 s8, s4, s1
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: s_cselect_b32 s1, s4, s1
-; GFX10-NEXT: s_cmp_eq_u32 s5, 0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s8, s1
-; GFX10-NEXT: s_cselect_b32 s6, s4, s0
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 1
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: s_cselect_b32 s0, s4, s0
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s6, s0
+; GFX10-NEXT: s_movrels_b32 s6, s0
+; GFX10-NEXT: s_andn2_b32 s5, s6, s7
+; GFX10-NEXT: s_or_b32 s4, s5, s4
+; GFX10-NEXT: s_movreld_b32 s0, s4
; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i16_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: s_cmp_eq_u32 s5, 6
+; GFX11-NEXT: s_lshr_b32 m0, s5, 1
+; GFX11-NEXT: s_and_b32 s5, s5, 1
+; GFX11-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX11-NEXT: s_lshl_b32 s5, s5, 4
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: s_lshl_b32 s7, 0xffff, s5
+; GFX11-NEXT: s_lshl_b32 s4, s4, s5
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cselect_b32 s6, s4, s3
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 7
-; GFX11-NEXT: s_cselect_b32 s3, s4, s3
-; GFX11-NEXT: s_cmp_eq_u32 s5, 4
-; GFX11-NEXT: s_pack_ll_b32_b16 s3, s6, s3
-; GFX11-NEXT: s_cselect_b32 s7, s4, s2
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 5
-; GFX11-NEXT: v_mov_b32_e32 v3, s3
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_cmp_eq_u32 s5, 2
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s7, s2
-; GFX11-NEXT: s_cselect_b32 s8, s4, s1
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 3
-; GFX11-NEXT: s_cselect_b32 s1, s4, s1
-; GFX11-NEXT: s_cmp_eq_u32 s5, 0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s8, s1
-; GFX11-NEXT: s_cselect_b32 s6, s4, s0
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 1
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: s_cselect_b32 s0, s4, s0
+; GFX11-NEXT: s_movrels_b32 s6, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s6, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_and_not1_b32 s5, s6, s7
+; GFX11-NEXT: s_or_b32 s4, s5, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_movreld_b32 s0, s4
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(4) %ptr
@@ -1550,87 +1735,43 @@ define amdgpu_ps void @insertelement_v_v8i16_s_s(ptr addrspace(1) %ptr, i16 inre
; GFX9-LABEL: insertelement_v_v8i16_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
-; GFX9-NEXT: s_cmp_eq_u32 s3, 6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 7
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 4
-; GFX9-NEXT: s_cselect_b64 s[14:15], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 5
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 2
+; GFX9-NEXT: s_and_b32 s1, s3, 1
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshr_b32 s0, s3, 1
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 3
-; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 0
-; GFX9-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 1
-; GFX9-NEXT: s_cselect_b64 s[12:13], -1, 0
-; GFX9-NEXT: s_mov_b32 s16, 0x5040100
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v3, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v6, v2, v4, s[14:15]
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_cndmask_b32_e64 v7, v1, v4, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_cndmask_b32_e64 v8, v0, v4, s[10:11]
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v3, v3, v5, s16
-; GFX9-NEXT: v_perm_b32 v2, v2, v6, s16
-; GFX9-NEXT: v_perm_b32 v1, v1, v7, s16
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v6, v5, s1, v4
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_perm_b32 v0, v0, v8, s16
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i16_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s3, 6
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 7
+; GFX8-NEXT: s_and_b32 s0, s3, 1
+; GFX8-NEXT: s_lshr_b32 m0, s3, 1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 4
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 5
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 3
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX8-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 0
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movrels_b32_e32 v4, v0
+; GFX8-NEXT: v_bfi_b32 v4, s0, 0, v4
+; GFX8-NEXT: v_or_b32_e32 v6, s1, v4
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movreld_b32_e32 v0, v6
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
@@ -1638,126 +1779,60 @@ define amdgpu_ps void @insertelement_v_v8i16_s_s(ptr addrspace(1) %ptr, i16 inre
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_cmp_eq_u32 s3, 6
-; GFX7-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 7
+; GFX7-NEXT: s_and_b32 s0, s3, 1
+; GFX7-NEXT: s_lshr_b32 m0, s3, 1
+; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 4
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 2
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 3
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 0
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v7, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 1
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT: v_movrels_b32_e32 v4, v0
+; GFX7-NEXT: v_bfi_b32 v4, s0, 0, v4
+; GFX7-NEXT: v_or_b32_e32 v4, s1, v4
+; GFX7-NEXT: v_movreld_b32_e32 v0, v4
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i16_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
-; GFX10-NEXT: s_cmp_eq_u32 s3, 6
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 7
+; GFX10-NEXT: s_and_b32 s0, s3, 1
+; GFX10-NEXT: s_lshr_b32 m0, s3, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v3, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 4
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 5
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s2, s1
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 2
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 3
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 1
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, s2, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
-; GFX10-NEXT: v_perm_b32 v3, v3, v4, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v6, v1, 0x5040100
+; GFX10-NEXT: v_movrels_b32_e32 v4, v0
+; GFX10-NEXT: v_and_or_b32 v6, v4, s1, s0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_perm_b32 v2, v5, v2, 0x5040100
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_perm_b32 v0, v7, v0, 0x5040100
+; GFX10-NEXT: v_movreld_b32_e32 v0, v6
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i16_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX11-NEXT: s_cmp_eq_u32 s3, 6
-; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 7
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 4
-; GFX11-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 5
-; GFX11-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 2
-; GFX11-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 3
-; GFX11-NEXT: s_cselect_b32 s7, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s8, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 1
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
+; GFX11-NEXT: s_and_b32 s0, s3, 1
+; GFX11-NEXT: s_lshr_b32 m0, s3, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: v_mov_b32_e32 v5, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v3.l, v3.l, s2, s0
-; GFX11-NEXT: v_cndmask_b16 v3.h, v3.h, s2, s1
-; GFX11-NEXT: v_cndmask_b16 v2.l, v2.l, s2, s4
-; GFX11-NEXT: v_cndmask_b16 v2.h, v2.h, s2, s5
-; GFX11-NEXT: v_cndmask_b16 v1.l, v1.l, s2, s6
-; GFX11-NEXT: v_cndmask_b16 v1.h, v1.h, s2, s7
-; GFX11-NEXT: v_cndmask_b16 v0.l, v0.l, s2, s8
-; GFX11-NEXT: v_cndmask_b16 v0.h, v0.h, s2, s3
+; GFX11-NEXT: v_movrels_b32_e32 v4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v6, v4, s1, s0
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v6
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(1 ) %ptr
@@ -1770,239 +1845,117 @@ define amdgpu_ps void @insertelement_s_v8i16_v_s(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v8i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: s_lshr_b32 s5, s4, 1
+; GFX9-NEXT: s_and_b32 s4, s4, 1
+; GFX9-NEXT: s_mov_b32 m0, s5
+; GFX9-NEXT: s_lshl_b32 s4, s4, 4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 7
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_mov_b32 s3, 0x5040100
-; GFX9-NEXT: s_cmp_eq_u32 s4, 4
-; GFX9-NEXT: v_perm_b32 v3, v2, v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_cmp_eq_u32 s4, 2
-; GFX9-NEXT: v_perm_b32 v2, v2, v1, s3
+; GFX9-NEXT: s_movrels_b32 s6, s0
+; GFX9-NEXT: s_lshl_b32 s7, 0xffff, s4
+; GFX9-NEXT: s_andn2_b32 s6, s6, s7
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_lshl_or_b32 v6, v0, s4, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: v_perm_b32 v1, v4, v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 1
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v4, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_perm_b32 v0, v0, v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: s_set_gpr_idx_on s5, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v8i16_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: s_cmp_eq_u32 s4, 6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: s_lshr_b32 m0, s4, 1
+; GFX8-NEXT: s_and_b32 s4, s4, 1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 4
+; GFX8-NEXT: v_mov_b32_e32 v1, s4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_cmp_eq_u32 s4, 7
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: s_cmp_eq_u32 s4, 4
-; GFX8-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_cmp_eq_u32 s4, 5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: s_cmp_eq_u32 s4, 2
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: s_movrels_b32 s5, s0
+; GFX8-NEXT: s_lshl_b32 s4, 0xffff, s4
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: s_andn2_b32 s4, s5, s4
+; GFX8-NEXT: v_or_b32_e32 v6, s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_cmp_eq_u32 s4, 3
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: s_cmp_eq_u32 s4, 0
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_cmp_eq_u32 s4, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v4, v0, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v4, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_movreld_b32_e32 v0, v6
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v8i16_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX7-NEXT: s_cmp_eq_u32 s4, 6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
+; GFX7-NEXT: s_lshr_b32 m0, s4, 1
+; GFX7-NEXT: s_and_b32 s4, s4, 1
+; GFX7-NEXT: s_lshl_b32 s4, s4, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: s_lshr_b32 s3, s3, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cmp_eq_u32 s4, 4
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s2, s2, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cmp_eq_u32 s4, 2
-; GFX7-NEXT: v_or_b32_e32 v2, v1, v2
+; GFX7-NEXT: s_movrels_b32 s5, s0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_lshl_b32 s4, 0xffff, s4
+; GFX7-NEXT: s_andn2_b32 s4, s5, s4
+; GFX7-NEXT: v_or_b32_e32 v4, s4, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 3
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: s_cmp_eq_u32 s4, 0
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 1
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: v_mov_b32_e32 v3, s3
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_movreld_b32_e32 v0, v4
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 6
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s5, s3, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 7
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s3, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 4
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s5, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s3, s2, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s2, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 2
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s3, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s1, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 3
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s1, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s2, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s1, s0, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 1
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s0, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_perm_b32 v3, v2, v1, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s1, v0, vcc_lo
-; GFX10-NEXT: v_perm_b32 v2, v5, v4, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v7, v6, 0x5040100
+; GFX10-NEXT: s_and_b32 s5, s4, 1
+; GFX10-NEXT: s_lshr_b32 m0, s4, 1
+; GFX10-NEXT: s_lshl_b32 s4, s5, 4
+; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x5040100
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_movrels_b32 s6, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_andn2_b32 s5, s6, s5
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_lshl_or_b32 v6, v0, s4, s5
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-NEXT: v_movreld_b32_e32 v0, v6
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 6
-; GFX11-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-NEXT: s_cselect_b32 s5, -1, 0
+; GFX11-NEXT: s_and_b32 s5, s4, 1
+; GFX11-NEXT: s_lshr_b32 m0, s4, 1
+; GFX11-NEXT: s_lshl_b32 s4, s5, 4
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_lshl_b32 s5, 0xffff, s4
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s6, s3, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 7
-; GFX11-NEXT: v_cndmask_b16 v3.l, s3, v0.h, s5
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 4
-; GFX11-NEXT: v_cndmask_b16 v3.h, s6, v0.h, s3
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_lshr_b32 s5, s2, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 5
-; GFX11-NEXT: v_cndmask_b16 v2.l, s2, v0.h, s3
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 2
-; GFX11-NEXT: v_cndmask_b16 v2.h, s5, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-NEXT: v_cndmask_b16 v1.l, s1, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: v_cndmask_b16 v1.h, s3, v0.h, s1
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 1
-; GFX11-NEXT: v_cndmask_b16 v0.l, s0, v0.h, s1
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b16 v0.h, s2, v0.h, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_movrels_b32 s6, s0
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_and_not1_b32 s5, s6, s5
+; GFX11-NEXT: v_mov_b32_e32 v3, s3
+; GFX11-NEXT: v_lshl_or_b32 v6, v0, s4, s5
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_movreld_b32_e32 v0, v6
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(4) %ptr
@@ -2015,203 +1968,164 @@ define amdgpu_ps void @insertelement_s_v8i16_s_v(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v8i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: s_mov_b32 s3, 0x5040100
-; GFX9-NEXT: v_perm_b32 v3, v2, v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_perm_b32 v2, v2, v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX9-NEXT: v_perm_b32 v1, v5, v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v5
+; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v3, s[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v5
+; GFX9-NEXT: v_lshlrev_b32_e64 v7, v0, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v8
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[2:3]
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_and_or_b32 v7, v6, v0, v7
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v1, v7, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v7, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_perm_b32 v0, v0, v6, s3
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, v7, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[5:6], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v8i16_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v5, v4, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v4, s3
+; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v5
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v3, s[0:1]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v5
+; GFX8-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX8-NEXT: v_mov_b32_e32 v8, 0xffff
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[2:3]
+; GFX8-NEXT: v_lshlrev_b32_e64 v7, v0, s4
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v8
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v6
+; GFX8-NEXT: v_or_b32_e32 v7, v0, v7
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v1, v7, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v3, v7, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v6, 0
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, v7, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[5:6], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v8i16_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 4, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: s_lshr_b32 s3, s3, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX7-NEXT: s_lshr_b32 s2, s2, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX7-NEXT: v_mov_b32_e32 v2, s1
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: v_mov_b32_e32 v4, s3
+; GFX7-NEXT: v_mov_b32_e32 v3, s2
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v5
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v3, s[0:1]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v5
+; GFX7-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[2:3]
+; GFX7-NEXT: v_lshl_b32_e32 v7, s4, v0
+; GFX7-NEXT: v_lshl_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v0, v7
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v1, v6, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v3, v6, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v4, v6, s[2:3]
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 1, v0
+; GFX10-NEXT: v_and_b32_e32 v4, 1, v0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 4, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
+; GFX10-NEXT: v_not_b32_e32 v7, v7
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX10-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v4, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
+; GFX10-NEXT: v_and_or_b32 v7, v5, v7, v4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s3, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s2, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s1, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s0, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT: v_perm_b32 v3, v3, v2, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s2, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT: v_perm_b32 v2, v9, v6, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v10, s1, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s0, v1, vcc_lo
-; GFX10-NEXT: v_perm_b32 v1, v10, v7, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x5040100
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i16_s_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v4.l, s4
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 2, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 0, v0
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 5, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 3, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 1, v0
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 1, v0
+; GFX11-NEXT: v_and_b32_e32 v4, 1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s7, s11, 16
-; GFX11-NEXT: v_cndmask_b16 v2.l, s10, v4.l, s1
-; GFX11-NEXT: s_lshr_b32 s1, s10, 16
-; GFX11-NEXT: v_cndmask_b16 v1.l, s9, v4.l, s3
-; GFX11-NEXT: s_lshr_b32 s3, s9, 16
-; GFX11-NEXT: v_cndmask_b16 v0.l, s8, v4.l, s5
-; GFX11-NEXT: s_lshr_b32 s5, s8, 16
-; GFX11-NEXT: v_cndmask_b16 v3.l, s11, v4.l, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v3.h, s7, v4.l, s0
-; GFX11-NEXT: v_cndmask_b16 v2.h, s1, v4.l, s2
-; GFX11-NEXT: v_cndmask_b16 v1.h, s3, v4.l, s4
-; GFX11-NEXT: v_cndmask_b16 v0.h, s5, v4.l, s6
-; GFX11-NEXT: global_store_b128 v[5:6], v[0:3], off
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 4, v4
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX11-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
+; GFX11-NEXT: v_lshlrev_b32_e64 v7, v4, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v4, s1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v7, v7
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v7, v5, v7, v4
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i16> %vec, i16 %val, i32 %idx
@@ -2223,199 +2137,162 @@ define amdgpu_ps void @insertelement_s_v8i16_v_v(ptr addrspace(4) inreg %ptr, i1
; GFX9-LABEL: insertelement_s_v8i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 1, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX9-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: s_mov_b32 s3, 0x5040100
-; GFX9-NEXT: v_perm_b32 v4, v3, v2, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_perm_b32 v3, v3, v2, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_perm_b32 v2, v5, v2, s3
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_perm_b32 v1, v0, v7, s3
-; GFX9-NEXT: global_store_dwordx4 v[5:6], v[1:4], off
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v5, s3
+; GFX9-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v6
+; GFX9-NEXT: v_mov_b32_e32 v8, 0xffff
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v4, s[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v6
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, v1, v8
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[2:3]
+; GFX9-NEXT: v_not_b32_e32 v1, v1
+; GFX9-NEXT: v_and_or_b32 v8, v7, v1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v8, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, v8, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v8, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[6:7], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v8i16_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX8-NEXT: v_mov_b32_e32 v6, 0
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 1, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v5, v0, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_or_b32_sdwa v1, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[5:6], v[1:4]
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v6
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v4, s[0:1]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v6
+; GFX8-NEXT: v_mov_b32_e32 v8, 0xffff
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[2:3]
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v1, v8
+; GFX8-NEXT: v_bfi_b32 v1, v1, 0, v7
+; GFX8-NEXT: v_or_b32_e32 v8, v1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v8, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, v8, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v6, 0
+; GFX8-NEXT: v_mov_b32_e32 v7, 0
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v8, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v8i16_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 1, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: s_lshr_b32 s3, s3, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX7-NEXT: s_lshr_b32 s2, s2, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v3, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v1, v5, v0
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_mov_b32_e32 v5, s3
+; GFX7-NEXT: v_mov_b32_e32 v4, s2
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v6
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v4, s[0:1]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v6
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[2:3]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, v7, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, v7, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v7, s[2:3]
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 1, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 1, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 4, v5
+; GFX10-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v5, v8
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s3, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v1
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s2, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v1
-; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s0, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v1
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s3, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v1
-; GFX10-NEXT: v_perm_b32 v3, v3, v2, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s2, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v1
-; GFX10-NEXT: v_perm_b32 v2, v9, v6, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v10, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX10-NEXT: v_perm_b32 v1, v10, v7, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s0, v0, vcc_lo
-; GFX10-NEXT: v_perm_b32 v0, v0, v8, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: v_mov_b32_e32 v4, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
+; GFX10-NEXT: v_and_or_b32 v8, v6, v5, v0
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
+; GFX10-NEXT: v_mov_b32_e32 v6, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
+; GFX10-NEXT: global_store_dwordx4 v[5:6], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i16_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 2, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 0, v1
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 5, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 3, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 1, v1
-; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v5, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s7, s11, 16
-; GFX11-NEXT: v_cndmask_b16 v2.l, s10, v0.h, s1
-; GFX11-NEXT: s_lshr_b32 s1, s10, 16
-; GFX11-NEXT: v_cndmask_b16 v1.l, s9, v0.h, s3
-; GFX11-NEXT: s_lshr_b32 s3, s9, 16
-; GFX11-NEXT: v_cndmask_b16 v0.l, s8, v0.h, s5
-; GFX11-NEXT: s_lshr_b32 s5, s8, 16
-; GFX11-NEXT: v_cndmask_b16 v3.l, s11, v0.h, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v3.h, s7, v0.h, s0
-; GFX11-NEXT: v_cndmask_b16 v2.h, s1, v0.h, s2
-; GFX11-NEXT: v_cndmask_b16 v1.h, s3, v0.h, s4
-; GFX11-NEXT: v_cndmask_b16 v0.h, s5, v0.h, s6
-; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v4, s3
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 4, v5
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
+; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v5, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v5, v8
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v8, v6, v5, v0
+; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
+; GFX11-NEXT: global_store_b128 v[5:6], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i16> %vec, i16 %val, i32 %idx
@@ -2427,171 +2304,150 @@ define amdgpu_ps void @insertelement_v_v8i16_s_v(ptr addrspace(1) %ptr, i16 inre
; GFX9-LABEL: insertelement_v_v8i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v2
-; GFX9-NEXT: s_mov_b32 s8, 0x5040100
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v9, v2, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v6, v0, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v6, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v5, v0, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v0, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_perm_b32 v6, v6, v1, s8
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_perm_b32 v5, v1, v7, s8
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v4, v1, v8, s8
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_perm_b32 v3, v3, v2, s8
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[2:3]
+; GFX9-NEXT: v_and_or_b32 v9, v2, v0, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v6, v9, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i16_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX8-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_e64 v9, v2, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, 0
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v6, v0, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_sdwa v6, v6, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v5, v0, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v0, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v6, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v5, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_or_b32_sdwa v5, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v4, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v3, v3, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v4, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[2:3]
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v2
+; GFX8-NEXT: v_or_b32_e32 v9, v0, v9
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v9, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[7:8], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v8i16_s_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v2
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, 0xf000
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 1, v2
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v2, s0, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: s_mov_b32 s10, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v5, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v3, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v7, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v8, v4
-; GFX7-NEXT: v_or_b32_e32 v0, v9, v0
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v6, s[2:3]
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v1, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v3, v7, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v5, v7, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[2:3]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v7, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v0, v0, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: v_not_b32_e32 v7, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v5, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v4, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v3, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v5, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v4, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_perm_b32 v2, v9, v6, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v11, v3, s2, vcc_lo
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v10, v7, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v11, v8, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v5, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s1
+; GFX10-NEXT: v_and_or_b32 v9, v2, v7, v0
+; GFX10-NEXT: v_mov_b32_e32 v7, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v9, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v9, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v9, s1
+; GFX10-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 2, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 3, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s7, 1, v2
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v2
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v7, v0, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v0, v0, s1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_not_b32_e32 v7, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v6.l, v6.l, s2, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v6.h, v6.h, s2, s0
-; GFX11-NEXT: v_cndmask_b16 v5.l, v5.l, s2, s1
-; GFX11-NEXT: v_cndmask_b16 v5.h, v5.h, s2, s3
-; GFX11-NEXT: v_cndmask_b16 v4.l, v4.l, s2, s4
-; GFX11-NEXT: v_cndmask_b16 v4.h, v4.h, s2, s5
-; GFX11-NEXT: v_cndmask_b16 v3.l, v3.l, s2, s6
-; GFX11-NEXT: v_cndmask_b16 v3.h, v3.h, s2, s7
-; GFX11-NEXT: global_store_b128 v[0:1], v[3:6], off
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v5, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v6, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v9, v2, v7, v0
+; GFX11-NEXT: v_mov_b32_e32 v7, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v3, v9, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v5, v9, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v6, v9, s1
+; GFX11-NEXT: global_store_b128 v[7:8], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(1) %ptr
%insert = insertelement <8 x i16> %vec, i16 %val, i32 %idx
@@ -2603,85 +2459,41 @@ define amdgpu_ps void @insertelement_v_v8i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX9-LABEL: insertelement_v_v8i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX9-NEXT: s_cmp_eq_u32 s2, 6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 7
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 4
-; GFX9-NEXT: s_cselect_b64 s[14:15], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 5
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 2
-; GFX9-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 3
-; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 0
-; GFX9-NEXT: s_mov_b32 s3, 0x5040100
-; GFX9-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 1
-; GFX9-NEXT: s_cselect_b64 s[12:13], -1, 0
+; GFX9-NEXT: s_and_b32 s1, s2, 1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshr_b32 s0, s2, 1
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, s1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v7, v5, v2, s[14:15]
-; GFX9-NEXT: v_perm_b32 v6, v1, v0, s3
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_perm_b32 v5, v0, v7, s3
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v4, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_cndmask_b32_e64 v8, v4, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e64 v9, v3, v2, s[10:11]
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v4, v0, v8, s3
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v2, v1, s1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_perm_b32 v3, v2, v9, s3
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i16_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s2, 6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 7
+; GFX8-NEXT: s_and_b32 s0, s2, 1
+; GFX8-NEXT: s_lshr_b32 m0, s2, 1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 4
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 5
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 3
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX8-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 0
-; GFX8-NEXT: v_or_b32_sdwa v5, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_cndmask_b32_sdwa v2, v3, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v4, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movrels_b32_e32 v1, v3
+; GFX8-NEXT: v_bfi_b32 v1, s0, 0, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v3, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movreld_b32_e32 v3, v2
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; GFX8-NEXT: s_endpgm
;
@@ -2689,125 +2501,60 @@ define amdgpu_ps void @insertelement_v_v8i16_v_s(ptr addrspace(1) %ptr, i16 %val
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_cmp_eq_u32 s2, 6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 7
+; GFX7-NEXT: s_and_b32 s0, s2, 1
+; GFX7-NEXT: s_lshr_b32 m0, s2, 1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 4
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 2
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 3
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v5
-; GFX7-NEXT: v_or_b32_e32 v5, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 0
-; GFX7-NEXT: v_or_b32_e32 v4, v9, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 1
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v3, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_or_b32_e32 v3, v0, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v8, v2, vcc
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v0, v1
-; GFX7-NEXT: buffer_store_dwordx4 v[2:5], off, s[4:7], 0
+; GFX7-NEXT: v_movrels_b32_e32 v1, v3
+; GFX7-NEXT: v_bfi_b32 v1, s0, 0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_movreld_b32_e32 v3, v0
+; GFX7-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX10-NEXT: s_cmp_eq_u32 s2, 6
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 7
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 4
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 5
-; GFX10-NEXT: s_cselect_b32 s3, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 2
-; GFX10-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 3
-; GFX10-NEXT: s_cselect_b32 s5, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 0
-; GFX10-NEXT: s_cselect_b32 s6, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 1
-; GFX10-NEXT: s_cselect_b32 s2, -1, 0
+; GFX10-NEXT: s_and_b32 s0, s2, 1
+; GFX10-NEXT: s_lshr_b32 m0, s2, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX10-NEXT: s_not_b32 s0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc_lo
-; GFX10-NEXT: s_mov_b32 vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v4, v2, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v6, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s3
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v5, v2, s1
-; GFX10-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s5
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v3, v2, s6
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, v4, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s2
-; GFX10-NEXT: v_cndmask_b32_sdwa v9, v3, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
-; GFX10-NEXT: v_perm_b32 v2, v5, v6, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v4, v7, 0x5040100
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_perm_b32 v0, v9, v8, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_movrels_b32_e32 v0, v3
+; GFX10-NEXT: v_and_or_b32 v2, v0, s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_movreld_b32_e32 v3, v2
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-NEXT: s_cmp_eq_u32 s2, 6
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 7
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 4
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 5
-; GFX11-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 2
-; GFX11-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 3
-; GFX11-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s7, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 1
+; GFX11-NEXT: s_lshr_b32 m0, s2, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
-; GFX11-NEXT: v_cndmask_b16 v6.h, v6.h, v2.l, s1
-; GFX11-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s3
-; GFX11-NEXT: v_cndmask_b16 v5.h, v5.h, v2.l, s4
-; GFX11-NEXT: v_cndmask_b16 v4.l, v4.l, v2.l, s5
-; GFX11-NEXT: v_cndmask_b16 v4.h, v4.h, v2.l, s6
-; GFX11-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s7
-; GFX11-NEXT: v_cndmask_b16 v3.h, v3.h, v2.l, s2
+; GFX11-NEXT: v_movrels_b32_e32 v1, v3
+; GFX11-NEXT: v_and_or_b32 v2, v1, s0, v0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_movreld_b32_e32 v3, v2
; GFX11-NEXT: global_store_b128 v[0:1], v[3:6], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(1) %ptr
@@ -2820,167 +2567,145 @@ define amdgpu_ps void @insertelement_v_v8i16_v_v(ptr addrspace(1) %ptr, i16 %val
; GFX9-LABEL: insertelement_v_v8i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v3
-; GFX9-NEXT: s_mov_b32 s8, 0x5040100
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v3, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: v_mov_b32_e32 v9, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v7, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v7, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v2, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_perm_b32 v7, v1, v0, s8
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_perm_b32 v6, v0, v8, s8
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v4, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v5, v0, v9, s8
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_perm_b32 v4, v2, v3, s8
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[4:7], off
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GFX9-NEXT: v_and_or_b32 v3, v3, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, v3, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v3, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i16_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX8-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v3, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v7, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v6, v2, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v5, v2, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v2, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v2, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v6, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_or_b32_sdwa v6, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v5, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v2, v4, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v5, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v4, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v0, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v4, v3, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v3, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v8i16_v_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
-; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v3
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, 0xf000
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v0
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: s_mov_b32 s10, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v4, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v0, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v7, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v8, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v9, v6
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v3
+; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v4, v3, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v3, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 6, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 5, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 1, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v3
+; GFX10-NEXT: v_mov_b32_e32 v9, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v8, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v2, v8
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v7, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v7, v2, s3
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v4, v2, vcc_lo
-; GFX10-NEXT: s_mov_b32 vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_sdwa v6, v6, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s1
-; GFX10-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s2
-; GFX10-NEXT: v_cndmask_b32_sdwa v10, v4, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v2, v6, v7, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v5, v8, 0x5040100
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_perm_b32 v0, v10, v9, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v6, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX10-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v4, v3, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s1
+; GFX10-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i16_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v3
-; GFX11-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 5, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 2, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 3, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 0, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 1, v3
-; GFX11-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v9, 0
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX11-NEXT: v_mov_b32_e32 v9, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 1, v3
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v3.l, v7.l, v0.h, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v3.h, v7.h, v0.h, s0
-; GFX11-NEXT: v_cndmask_b16 v2.l, v6.l, v0.h, s1
-; GFX11-NEXT: v_cndmask_b16 v2.h, v6.h, v0.h, s2
-; GFX11-NEXT: v_cndmask_b16 v1.l, v5.l, v0.h, s3
-; GFX11-NEXT: v_cndmask_b16 v1.h, v5.h, v0.h, s4
-; GFX11-NEXT: v_cndmask_b16 v0.l, v4.l, v0.h, s5
-; GFX11-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s6
+; GFX11-NEXT: v_dual_cndmask_b32 v3, v4, v5 :: v_dual_lshlrev_b32 v0, 4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v6, s0
+; GFX11-NEXT: v_lshlrev_b32_e64 v8, v0, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: v_not_b32_e32 v2, v8
+; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, v3, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s1
; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i16>, ptr addrspace(1) %ptr
@@ -2993,372 +2718,148 @@ define amdgpu_ps void @insertelement_s_v16i16_s_s(ptr addrspace(4) inreg %ptr, i
; GFX9-LABEL: insertelement_s_v16i16_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX9-NEXT: s_cmp_eq_u32 s5, 6
-; GFX9-NEXT: v_mov_b32_e32 v4, 16
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: s_and_b32 s1, s5, 1
+; GFX9-NEXT: s_lshr_b32 m0, s5, 1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_and_b32 s2, s4, 0xffff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cselect_b32 s0, s4, s11
-; GFX9-NEXT: s_lshr_b32 s1, s11, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 7
-; GFX9-NEXT: s_cselect_b32 s1, s4, s1
-; GFX9-NEXT: s_cmp_eq_u32 s5, 4
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX9-NEXT: s_cselect_b32 s1, s4, s10
-; GFX9-NEXT: s_lshr_b32 s2, s10, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 5
-; GFX9-NEXT: s_cselect_b32 s2, s4, s2
-; GFX9-NEXT: s_cmp_eq_u32 s5, 2
-; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s2
-; GFX9-NEXT: s_cselect_b32 s2, s4, s9
-; GFX9-NEXT: s_lshr_b32 s3, s9, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 3
-; GFX9-NEXT: s_cselect_b32 s3, s4, s3
-; GFX9-NEXT: s_cmp_eq_u32 s5, 0
-; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX9-NEXT: s_cselect_b32 s3, s4, s8
-; GFX9-NEXT: s_lshr_b32 s6, s8, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 1
-; GFX9-NEXT: s_cselect_b32 s6, s4, s6
-; GFX9-NEXT: s_cmp_eq_u32 s5, 14
-; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s6
-; GFX9-NEXT: s_cselect_b32 s6, s4, s15
-; GFX9-NEXT: s_lshr_b32 s7, s15, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 15
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_cmp_eq_u32 s5, 12
-; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s7
-; GFX9-NEXT: s_cselect_b32 s7, s4, s14
-; GFX9-NEXT: s_lshr_b32 s8, s14, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 13
-; GFX9-NEXT: s_cselect_b32 s8, s4, s8
-; GFX9-NEXT: s_cmp_eq_u32 s5, 10
-; GFX9-NEXT: s_pack_ll_b32_b16 s7, s7, s8
-; GFX9-NEXT: s_cselect_b32 s8, s4, s13
-; GFX9-NEXT: s_lshr_b32 s9, s13, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 11
-; GFX9-NEXT: s_cselect_b32 s9, s4, s9
-; GFX9-NEXT: s_cmp_eq_u32 s5, 8
-; GFX9-NEXT: s_pack_ll_b32_b16 s8, s8, s9
-; GFX9-NEXT: s_cselect_b32 s9, s4, s12
-; GFX9-NEXT: s_lshr_b32 s10, s12, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 9
-; GFX9-NEXT: s_cselect_b32 s4, s4, s10
-; GFX9-NEXT: s_pack_ll_b32_b16 s4, s9, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX9-NEXT: s_movrels_b32 s0, s8
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: s_andn2_b32 s0, s0, s1
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_movreld_b32 s8, s0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX9-NEXT: v_mov_b32_e32 v4, 16
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s12
+; GFX9-NEXT: v_mov_b32_e32 v1, s13
+; GFX9-NEXT: v_mov_b32_e32 v2, s14
+; GFX9-NEXT: v_mov_b32_e32 v3, s15
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i16_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX8-NEXT: s_cmp_eq_u32 s5, 6
-; GFX8-NEXT: v_mov_b32_e32 v4, 16
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: s_and_b32 s1, s5, 1
+; GFX8-NEXT: s_lshr_b32 m0, s5, 1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 4
+; GFX8-NEXT: s_and_b32 s2, s4, 0xffff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_cselect_b32 s0, s4, s11
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_lshr_b32 s1, s11, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 7
-; GFX8-NEXT: s_cselect_b32 s1, s4, s1
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_cmp_eq_u32 s5, 4
-; GFX8-NEXT: s_cselect_b32 s1, s4, s10
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s2, s10, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 5
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_or_b32 s1, s1, s2
-; GFX8-NEXT: s_cmp_eq_u32 s5, 2
-; GFX8-NEXT: s_cselect_b32 s2, s4, s9
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_lshr_b32 s3, s9, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 3
-; GFX8-NEXT: s_cselect_b32 s3, s4, s3
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_or_b32 s2, s2, s3
-; GFX8-NEXT: s_cmp_eq_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s3, s4, s8
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshr_b32 s6, s8, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 1
-; GFX8-NEXT: s_cselect_b32 s6, s4, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_or_b32 s3, s3, s6
-; GFX8-NEXT: s_cmp_eq_u32 s5, 14
-; GFX8-NEXT: s_cselect_b32 s6, s4, s15
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshr_b32 s7, s15, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 15
-; GFX8-NEXT: s_cselect_b32 s7, s4, s7
-; GFX8-NEXT: s_lshl_b32 s7, s7, 16
-; GFX8-NEXT: s_or_b32 s6, s6, s7
-; GFX8-NEXT: s_cmp_eq_u32 s5, 12
-; GFX8-NEXT: s_cselect_b32 s7, s4, s14
-; GFX8-NEXT: s_and_b32 s7, s7, 0xffff
-; GFX8-NEXT: s_lshr_b32 s8, s14, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 13
-; GFX8-NEXT: s_cselect_b32 s8, s4, s8
-; GFX8-NEXT: s_lshl_b32 s8, s8, 16
-; GFX8-NEXT: s_or_b32 s7, s7, s8
-; GFX8-NEXT: s_cmp_eq_u32 s5, 10
-; GFX8-NEXT: s_cselect_b32 s8, s4, s13
-; GFX8-NEXT: s_and_b32 s8, s8, 0xffff
-; GFX8-NEXT: s_lshr_b32 s9, s13, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 11
-; GFX8-NEXT: s_cselect_b32 s9, s4, s9
-; GFX8-NEXT: s_lshl_b32 s9, s9, 16
-; GFX8-NEXT: s_or_b32 s8, s8, s9
-; GFX8-NEXT: s_cmp_eq_u32 s5, 8
-; GFX8-NEXT: s_cselect_b32 s9, s4, s12
-; GFX8-NEXT: s_and_b32 s9, s9, 0xffff
-; GFX8-NEXT: s_lshr_b32 s10, s12, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 9
-; GFX8-NEXT: s_cselect_b32 s4, s4, s10
-; GFX8-NEXT: s_lshl_b32 s4, s4, 16
-; GFX8-NEXT: s_or_b32 s4, s9, s4
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: v_mov_b32_e32 v2, s7
-; GFX8-NEXT: v_mov_b32_e32 v3, s6
-; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: s_movrels_b32 s0, s8
+; GFX8-NEXT: s_lshl_b32 s2, s2, s1
+; GFX8-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_andn2_b32 s0, s0, s1
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_movreld_b32 s8, s0
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: v_mov_b32_e32 v2, s10
+; GFX8-NEXT: v_mov_b32_e32 v3, s11
+; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v4, 16
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_mov_b32_e32 v1, s13
+; GFX8-NEXT: v_mov_b32_e32 v2, s14
+; GFX8-NEXT: v_mov_b32_e32 v3, s15
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i16_s_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX7-NEXT: s_cmp_eq_u32 s5, 6
+; GFX7-NEXT: s_and_b32 s1, s5, 1
+; GFX7-NEXT: s_lshr_b32 m0, s5, 1
+; GFX7-NEXT: s_lshl_b32 s1, s1, 4
+; GFX7-NEXT: s_and_b32 s2, s4, 0xffff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_cselect_b32 s0, s4, s11
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_lshr_b32 s1, s11, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 7
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 16
-; GFX7-NEXT: s_or_b32 s6, s0, s1
-; GFX7-NEXT: s_cmp_eq_u32 s5, 4
-; GFX7-NEXT: s_cselect_b32 s0, s4, s10
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_lshr_b32 s1, s10, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 5
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 16
-; GFX7-NEXT: s_or_b32 s7, s0, s1
-; GFX7-NEXT: s_cmp_eq_u32 s5, 2
-; GFX7-NEXT: s_cselect_b32 s0, s4, s9
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_lshr_b32 s1, s9, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 3
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 16
-; GFX7-NEXT: s_or_b32 s9, s0, s1
-; GFX7-NEXT: s_cmp_eq_u32 s5, 0
-; GFX7-NEXT: s_cselect_b32 s0, s4, s8
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_lshr_b32 s1, s8, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 1
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 16
-; GFX7-NEXT: s_or_b32 s8, s0, s1
-; GFX7-NEXT: s_cmp_eq_u32 s5, 14
-; GFX7-NEXT: s_cselect_b32 s0, s4, s15
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_lshr_b32 s1, s15, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 15
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 16
-; GFX7-NEXT: s_or_b32 s0, s0, s1
-; GFX7-NEXT: s_cmp_eq_u32 s5, 12
-; GFX7-NEXT: s_cselect_b32 s1, s4, s14
-; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX7-NEXT: s_lshr_b32 s2, s14, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 13
-; GFX7-NEXT: s_cselect_b32 s2, s4, s2
-; GFX7-NEXT: s_lshl_b32 s2, s2, 16
-; GFX7-NEXT: s_or_b32 s1, s1, s2
-; GFX7-NEXT: s_cmp_eq_u32 s5, 10
-; GFX7-NEXT: s_cselect_b32 s2, s4, s13
-; GFX7-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX7-NEXT: s_lshr_b32 s3, s13, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 11
-; GFX7-NEXT: s_cselect_b32 s3, s4, s3
-; GFX7-NEXT: s_lshl_b32 s3, s3, 16
-; GFX7-NEXT: s_or_b32 s2, s2, s3
-; GFX7-NEXT: s_cmp_eq_u32 s5, 8
-; GFX7-NEXT: s_cselect_b32 s3, s4, s12
-; GFX7-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX7-NEXT: s_lshr_b32 s10, s12, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 9
-; GFX7-NEXT: s_cselect_b32 s4, s4, s10
-; GFX7-NEXT: s_lshl_b32 s4, s4, 16
-; GFX7-NEXT: s_or_b32 s3, s3, s4
-; GFX7-NEXT: v_mov_b32_e32 v0, s3
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_mov_b32_e32 v2, s1
-; GFX7-NEXT: v_mov_b32_e32 v3, s0
-; GFX7-NEXT: s_mov_b64 s[0:1], 16
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_movrels_b32 s0, s8
+; GFX7-NEXT: s_lshl_b32 s2, s2, s1
+; GFX7-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX7-NEXT: s_andn2_b32 s0, s0, s1
+; GFX7-NEXT: s_or_b32 s0, s0, s2
+; GFX7-NEXT: s_movreld_b32 s8, s0
; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: v_mov_b32_e32 v0, s8
; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_mov_b32_e32 v2, s7
-; GFX7-NEXT: v_mov_b32_e32 v3, s6
+; GFX7-NEXT: v_mov_b32_e32 v2, s10
+; GFX7-NEXT: v_mov_b32_e32 v3, s11
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 16
+; GFX7-NEXT: v_mov_b32_e32 v0, s12
+; GFX7-NEXT: v_mov_b32_e32 v1, s13
+; GFX7-NEXT: v_mov_b32_e32 v2, s14
+; GFX7-NEXT: v_mov_b32_e32 v3, s15
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i16_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX10-NEXT: s_cmp_eq_u32 s5, 6
-; GFX10-NEXT: v_mov_b32_e32 v8, 16
+; GFX10-NEXT: s_and_b32 s0, s5, 1
+; GFX10-NEXT: s_lshr_b32 m0, s5, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: v_mov_b32_e32 v9, 0
-; GFX10-NEXT: v_mov_b32_e32 v10, 0
+; GFX10-NEXT: v_mov_b32_e32 v10, 16
; GFX10-NEXT: v_mov_b32_e32 v11, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cselect_b32 s0, s4, s11
-; GFX10-NEXT: s_lshr_b32 s1, s11, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 7
-; GFX10-NEXT: s_cselect_b32 s1, s4, s1
-; GFX10-NEXT: s_cmp_eq_u32 s5, 4
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX10-NEXT: s_cselect_b32 s2, s4, s10
-; GFX10-NEXT: s_lshr_b32 s3, s10, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 5
-; GFX10-NEXT: v_mov_b32_e32 v3, s0
-; GFX10-NEXT: s_cselect_b32 s3, s4, s3
-; GFX10-NEXT: s_cmp_eq_u32 s5, 2
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX10-NEXT: s_cselect_b32 s6, s4, s9
-; GFX10-NEXT: s_lshr_b32 s7, s9, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: s_cselect_b32 s1, s4, s7
-; GFX10-NEXT: s_cmp_eq_u32 s5, 0
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX10-NEXT: s_cselect_b32 s0, s4, s8
-; GFX10-NEXT: s_lshr_b32 s7, s8, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 1
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: s_cselect_b32 s3, s4, s7
-; GFX10-NEXT: s_cmp_eq_u32 s5, 14
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX10-NEXT: s_cselect_b32 s2, s4, s15
-; GFX10-NEXT: s_lshr_b32 s7, s15, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 15
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_cselect_b32 s6, s4, s7
-; GFX10-NEXT: s_cmp_eq_u32 s5, 12
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s2, s6
-; GFX10-NEXT: s_cselect_b32 s1, s4, s14
-; GFX10-NEXT: s_lshr_b32 s7, s14, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 13
-; GFX10-NEXT: v_mov_b32_e32 v7, s2
-; GFX10-NEXT: s_cselect_b32 s3, s4, s7
-; GFX10-NEXT: s_cmp_eq_u32 s5, 10
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s3
-; GFX10-NEXT: s_cselect_b32 s6, s4, s13
-; GFX10-NEXT: s_lshr_b32 s7, s13, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 11
-; GFX10-NEXT: v_mov_b32_e32 v6, s1
-; GFX10-NEXT: s_cselect_b32 s2, s4, s7
-; GFX10-NEXT: s_cmp_eq_u32 s5, 8
-; GFX10-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX10-NEXT: s_cselect_b32 s3, s4, s12
-; GFX10-NEXT: s_lshr_b32 s7, s12, 16
-; GFX10-NEXT: s_cmp_eq_u32 s5, 9
-; GFX10-NEXT: v_mov_b32_e32 v5, s2
-; GFX10-NEXT: s_cselect_b32 s1, s4, s7
-; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
-; GFX10-NEXT: v_mov_b32_e32 v4, s1
-; GFX10-NEXT: global_store_dwordx4 v[8:9], v[4:7], off
-; GFX10-NEXT: global_store_dwordx4 v[10:11], v[0:3], off
+; GFX10-NEXT: s_movrels_b32 s3, s8
+; GFX10-NEXT: s_andn2_b32 s1, s3, s2
+; GFX10-NEXT: s_or_b32 s0, s1, s0
+; GFX10-NEXT: s_movreld_b32 s8, s0
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: v_mov_b32_e32 v2, s10
+; GFX10-NEXT: v_mov_b32_e32 v3, s11
+; GFX10-NEXT: v_mov_b32_e32 v4, s12
+; GFX10-NEXT: v_mov_b32_e32 v5, s13
+; GFX10-NEXT: v_mov_b32_e32 v6, s14
+; GFX10-NEXT: v_mov_b32_e32 v7, s15
+; GFX10-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i16_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[8:15], s[2:3], 0x0
-; GFX11-NEXT: s_cmp_eq_u32 s5, 6
-; GFX11-NEXT: v_dual_mov_b32 v8, 16 :: v_dual_mov_b32 v9, 0
-; GFX11-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0
+; GFX11-NEXT: s_and_b32 s0, s5, 1
+; GFX11-NEXT: s_lshr_b32 m0, s5, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v9, 0
+; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v11, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cselect_b32 s0, s4, s11
-; GFX11-NEXT: s_lshr_b32 s1, s11, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 7
-; GFX11-NEXT: s_cselect_b32 s1, s4, s1
-; GFX11-NEXT: s_cmp_eq_u32 s5, 4
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s1
-; GFX11-NEXT: s_cselect_b32 s2, s4, s10
-; GFX11-NEXT: s_lshr_b32 s3, s10, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 5
-; GFX11-NEXT: s_cselect_b32 s3, s4, s3
-; GFX11-NEXT: s_cmp_eq_u32 s5, 2
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s3
-; GFX11-NEXT: s_cselect_b32 s6, s4, s9
-; GFX11-NEXT: s_lshr_b32 s7, s9, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 3
-; GFX11-NEXT: v_dual_mov_b32 v3, s0 :: v_dual_mov_b32 v2, s2
-; GFX11-NEXT: s_cselect_b32 s1, s4, s7
-; GFX11-NEXT: s_cmp_eq_u32 s5, 0
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s6, s1
-; GFX11-NEXT: s_cselect_b32 s0, s4, s8
-; GFX11-NEXT: s_lshr_b32 s7, s8, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 1
-; GFX11-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-NEXT: s_cselect_b32 s3, s4, s7
-; GFX11-NEXT: s_cmp_eq_u32 s5, 14
-; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s3
-; GFX11-NEXT: s_cselect_b32 s2, s4, s15
-; GFX11-NEXT: s_lshr_b32 s7, s15, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 15
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: s_cselect_b32 s6, s4, s7
-; GFX11-NEXT: s_cmp_eq_u32 s5, 12
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s6
-; GFX11-NEXT: s_cselect_b32 s1, s4, s14
-; GFX11-NEXT: s_lshr_b32 s7, s14, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 13
-; GFX11-NEXT: v_mov_b32_e32 v7, s2
-; GFX11-NEXT: s_cselect_b32 s3, s4, s7
-; GFX11-NEXT: s_cmp_eq_u32 s5, 10
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s3
-; GFX11-NEXT: s_cselect_b32 s6, s4, s13
-; GFX11-NEXT: s_lshr_b32 s7, s13, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 11
-; GFX11-NEXT: v_mov_b32_e32 v6, s1
-; GFX11-NEXT: s_cselect_b32 s2, s4, s7
-; GFX11-NEXT: s_cmp_eq_u32 s5, 8
-; GFX11-NEXT: s_pack_ll_b32_b16 s2, s6, s2
-; GFX11-NEXT: s_cselect_b32 s3, s4, s12
-; GFX11-NEXT: s_lshr_b32 s7, s12, 16
-; GFX11-NEXT: s_cmp_eq_u32 s5, 9
-; GFX11-NEXT: v_mov_b32_e32 v5, s2
-; GFX11-NEXT: s_cselect_b32 s1, s4, s7
+; GFX11-NEXT: s_movrels_b32 s3, s8
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_pack_ll_b32_b16 s1, s3, s1
-; GFX11-NEXT: v_mov_b32_e32 v4, s1
+; GFX11-NEXT: s_and_not1_b32 s1, s3, s2
+; GFX11-NEXT: s_or_b32 s0, s1, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_movreld_b32 s8, s0
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
+; GFX11-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v3, s11
+; GFX11-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v5, s13
+; GFX11-NEXT: v_dual_mov_b32 v6, s14 :: v_dual_mov_b32 v7, s15
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[8:9], v[4:7], off
-; GFX11-NEXT: global_store_b128 v[10:11], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[10:11], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -3371,265 +2872,78 @@ define amdgpu_ps void @insertelement_v_v16i16_s_s(ptr addrspace(1) %ptr, i16 inr
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
; GFX9-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16
-; GFX9-NEXT: s_cmp_eq_u32 s3, 6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 7
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 4
-; GFX9-NEXT: s_cselect_b64 s[30:31], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 5
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 2
-; GFX9-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 3
-; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 0
-; GFX9-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 1
-; GFX9-NEXT: s_cselect_b64 s[12:13], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 14
-; GFX9-NEXT: v_mov_b32_e32 v14, s2
-; GFX9-NEXT: s_cselect_b64 s[14:15], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 15
-; GFX9-NEXT: s_cselect_b64 s[16:17], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 12
-; GFX9-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 13
-; GFX9-NEXT: s_cselect_b64 s[20:21], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 10
-; GFX9-NEXT: s_cselect_b64 s[22:23], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 11
-; GFX9-NEXT: s_cselect_b64 s[24:25], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 8
-; GFX9-NEXT: s_mov_b32 s33, 0x5040100
-; GFX9-NEXT: s_cselect_b64 s[26:27], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 9
-; GFX9-NEXT: s_cselect_b64 s[28:29], -1, 0
+; GFX9-NEXT: s_and_b32 s1, s3, 1
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshr_b32 s0, s3, 1
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: s_not_b32 s1, s1
+; GFX9-NEXT: v_mov_b32_e32 v12, s2
; GFX9-NEXT: v_mov_b32_e32 v10, 16
; GFX9-NEXT: v_mov_b32_e32 v11, 0
-; GFX9-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-NEXT: v_mov_b32_e32 v13, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v5, v14, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v5, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v5, v4, v14, s[30:31]
-; GFX9-NEXT: v_cndmask_b32_sdwa v4, v4, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_cndmask_b32_sdwa v17, v3, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_cndmask_b32_sdwa v18, v2, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[16:17]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e64 v19, v9, v14, s[14:15]
-; GFX9-NEXT: v_cndmask_b32_sdwa v9, v9, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[20:21]
-; GFX9-NEXT: v_cndmask_b32_e64 v16, v2, v14, s[10:11]
-; GFX9-NEXT: v_perm_b32 v2, v4, v5, s33
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v14, s[18:19]
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v8, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[24:25]
-; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v14, s[22:23]
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v7, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[28:29]
-; GFX9-NEXT: v_perm_b32 v8, v8, v4, s33
-; GFX9-NEXT: v_cndmask_b32_e64 v4, v6, v14, s[26:27]
-; GFX9-NEXT: v_perm_b32 v7, v7, v5, s33
-; GFX9-NEXT: v_cndmask_b32_sdwa v5, v6, v14, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e64 v15, v3, v14, s[6:7]
-; GFX9-NEXT: v_perm_b32 v9, v9, v19, s33
-; GFX9-NEXT: v_perm_b32 v6, v5, v4, s33
-; GFX9-NEXT: v_perm_b32 v3, v1, v0, s33
-; GFX9-NEXT: v_perm_b32 v1, v17, v15, s33
-; GFX9-NEXT: v_perm_b32 v0, v18, v16, s33
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v13, v2
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v12, v13, s1, v12
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v2, v12
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
; GFX9-NEXT: global_store_dwordx4 v[10:11], v[6:9], off
-; GFX9-NEXT: global_store_dwordx4 v[12:13], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i16_s_s:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 16, v0
-; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx4 v[2:5], v[2:3]
-; GFX8-NEXT: flat_load_dwordx4 v[6:9], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s3, 14
-; GFX8-NEXT: v_mov_b32_e32 v14, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 15
-; GFX8-NEXT: v_mov_b32_e32 v12, 0
-; GFX8-NEXT: v_mov_b32_e32 v13, 0
+; GFX8-NEXT: v_add_u32_e32 v4, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-NEXT: s_and_b32 s0, s3, 1
+; GFX8-NEXT: s_lshr_b32 m0, s3, 1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
; GFX8-NEXT: v_mov_b32_e32 v10, 16
; GFX8-NEXT: v_mov_b32_e32 v11, 0
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 12
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 13
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v14, s[0:1]
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 10
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 11
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 8
-; GFX8-NEXT: v_cndmask_b32_e64 v17, v3, v14, s[4:5]
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v14, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e64 v18, v2, v14, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v14, s[0:1]
-; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX8-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v2, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v15, v14, s[6:7]
-; GFX8-NEXT: s_cmp_eq_u32 s3, 9
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 6
-; GFX8-NEXT: v_or_b32_sdwa v1, v17, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v16, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 7
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v19, 16, v9
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 4
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v19, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 5
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v14, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s3, 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 3
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v7
-; GFX8-NEXT: v_or_b32_sdwa v8, v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v7, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 1
-; GFX8-NEXT: v_or_b32_sdwa v9, v9, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v6, v14, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_or_b32_sdwa v7, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v4, v6, v14, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_or_b32_sdwa v6, v15, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v18, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[12:13], v[6:9]
-; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
+; GFX8-NEXT: v_movrels_b32_e32 v12, v0
+; GFX8-NEXT: v_bfi_b32 v12, s0, 0, v12
+; GFX8-NEXT: v_or_b32_e32 v12, s1, v12
+; GFX8-NEXT: v_movreld_b32_e32 v0, v12
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v16i16_s_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx4 v[2:5], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: buffer_load_dwordx4 v[6:9], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-NEXT: s_cmp_eq_u32 s3, 6
-; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 7
-; GFX7-NEXT: s_mov_b64 s[4:5], 16
+; GFX7-NEXT: s_and_b32 s0, s3, 1
+; GFX7-NEXT: s_lshr_b32 m0, s3, 1
+; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v0, vcc
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 2
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 3
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 0
-; GFX7-NEXT: v_or_b32_e32 v5, v1, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v11, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 1
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 14
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v12, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 15
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v9
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 12
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v13, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 13
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v8
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 10
-; GFX7-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v14, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 11
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v7
-; GFX7-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 8
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX7-NEXT: v_or_b32_e32 v8, v8, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v15, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 9
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v7, v7, v10
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v0
-; GFX7-NEXT: buffer_store_dwordx4 v[6:9], off, s[4:7], 0
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_movrels_b32_e32 v0, v2
+; GFX7-NEXT: v_bfi_b32 v0, s0, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, s1, v0
+; GFX7-NEXT: v_movreld_b32_e32 v2, v0
; GFX7-NEXT: buffer_store_dwordx4 v[2:5], off, s[4:7], 0
+; GFX7-NEXT: s_mov_b64 s[4:5], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[6:9], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i16_s_s:
@@ -3637,78 +2951,23 @@ define amdgpu_ps void @insertelement_v_v16i16_s_s(ptr addrspace(1) %ptr, i16 inr
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: global_load_dwordx4 v[2:5], v[0:1], off
; GFX10-NEXT: global_load_dwordx4 v[6:9], v[0:1], off offset:16
-; GFX10-NEXT: s_cmp_eq_u32 s3, 6
+; GFX10-NEXT: s_and_b32 s0, s3, 1
+; GFX10-NEXT: s_lshr_b32 m0, s3, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mov_b32_e32 v10, 16
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 7
; GFX10-NEXT: v_mov_b32_e32 v11, 0
-; GFX10-NEXT: v_mov_b32_e32 v12, 0
-; GFX10-NEXT: v_mov_b32_e32 v13, 0
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v5, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 4
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v5
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 5
-; GFX10-NEXT: v_cndmask_b32_e64 v20, v4, s2, s1
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 2
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 3
-; GFX10-NEXT: v_cndmask_b32_e64 v21, v3, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 1
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, s2, s1
-; GFX10-NEXT: v_perm_b32 v4, v1, v0, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 14
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v16, 16, v9
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v15, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 15
-; GFX10-NEXT: v_perm_b32 v3, v5, v20, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v9, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 12
-; GFX10-NEXT: v_lshrrev_b32_e32 v17, 16, v8
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v16, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 13
-; GFX10-NEXT: v_cndmask_b32_e64 v15, v8, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 10
-; GFX10-NEXT: v_lshrrev_b32_e32 v18, 16, v7
-; GFX10-NEXT: v_cndmask_b32_e64 v16, v17, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 11
-; GFX10-NEXT: v_cndmask_b32_e64 v17, v7, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 8
-; GFX10-NEXT: v_lshrrev_b32_e32 v19, 16, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v18, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 9
-; GFX10-NEXT: v_cndmask_b32_e64 v20, v6, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: v_perm_b32 v8, v9, v5, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v19, v19, s2, s0
-; GFX10-NEXT: v_perm_b32 v7, v16, v15, 0x5040100
-; GFX10-NEXT: v_perm_b32 v6, v18, v17, 0x5040100
-; GFX10-NEXT: v_perm_b32 v2, v14, v21, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v1, v0, 0x5040100
-; GFX10-NEXT: v_perm_b32 v5, v19, v20, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[10:11], v[5:8], off
-; GFX10-NEXT: global_store_dwordx4 v[12:13], v[1:4], off
+; GFX10-NEXT: v_movrels_b32_e32 v0, v2
+; GFX10-NEXT: v_and_or_b32 v12, v0, s1, s0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_movreld_b32_e32 v2, v12
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX10-NEXT: global_store_dwordx4 v[10:11], v[6:9], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v16i16_s_s:
@@ -3716,61 +2975,24 @@ define amdgpu_ps void @insertelement_v_v16i16_s_s(ptr addrspace(1) %ptr, i16 inr
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b128 v[2:5], v[0:1], off
; GFX11-NEXT: global_load_b128 v[6:9], v[0:1], off offset:16
-; GFX11-NEXT: s_cmp_eq_u32 s3, 6
-; GFX11-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 7
-; GFX11-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 4
-; GFX11-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 5
-; GFX11-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 2
-; GFX11-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 3
-; GFX11-NEXT: s_cselect_b32 s7, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 0
-; GFX11-NEXT: s_cselect_b32 s8, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 1
-; GFX11-NEXT: s_cselect_b32 s9, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 14
-; GFX11-NEXT: s_cselect_b32 s10, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 15
-; GFX11-NEXT: s_cselect_b32 s11, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 12
-; GFX11-NEXT: s_cselect_b32 s12, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 13
-; GFX11-NEXT: s_cselect_b32 s13, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 10
-; GFX11-NEXT: s_cselect_b32 s14, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 11
-; GFX11-NEXT: s_cselect_b32 s15, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 8
-; GFX11-NEXT: s_cselect_b32 s16, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s3, 9
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_cndmask_b16 v5.l, v5.l, s2, s0
+; GFX11-NEXT: s_and_b32 s0, s3, 1
+; GFX11-NEXT: s_lshr_b32 m0, s3, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX11-NEXT: s_lshl_b32 s2, 0xffff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v11, 0
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v9.l, v9.l, s2, s10
-; GFX11-NEXT: v_cndmask_b16 v9.h, v9.h, s2, s11
-; GFX11-NEXT: v_cndmask_b16 v8.l, v8.l, s2, s12
-; GFX11-NEXT: v_cndmask_b16 v8.h, v8.h, s2, s13
-; GFX11-NEXT: v_cndmask_b16 v7.l, v7.l, s2, s14
-; GFX11-NEXT: v_cndmask_b16 v7.h, v7.h, s2, s15
-; GFX11-NEXT: v_cndmask_b16 v6.l, v6.l, s2, s16
-; GFX11-NEXT: v_cndmask_b16 v6.h, v6.h, s2, s3
-; GFX11-NEXT: v_cndmask_b16 v5.h, v5.h, s2, s1
-; GFX11-NEXT: v_cndmask_b16 v4.l, v4.l, s2, s4
-; GFX11-NEXT: v_cndmask_b16 v4.h, v4.h, s2, s5
-; GFX11-NEXT: v_cndmask_b16 v3.l, v3.l, s2, s6
-; GFX11-NEXT: v_cndmask_b16 v3.h, v3.h, s2, s7
-; GFX11-NEXT: v_cndmask_b16 v2.l, v2.l, s2, s8
-; GFX11-NEXT: v_cndmask_b16 v2.h, v2.h, s2, s9
+; GFX11-NEXT: v_movrels_b32_e32 v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v12, v0, s1, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: v_movreld_b32_e32 v2, v12
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off
-; GFX11-NEXT: global_store_b128 v[10:11], v[2:5], off
+; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off
+; GFX11-NEXT: global_store_b128 v[10:11], v[6:9], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(1 ) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -3782,446 +3004,149 @@ define amdgpu_ps void @insertelement_s_v16i16_v_s(ptr addrspace(4) inreg %ptr, i
; GFX9-LABEL: insertelement_s_v16i16_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: s_lshr_b32 s0, s4, 1
+; GFX9-NEXT: s_and_b32 s2, s4, 1
+; GFX9-NEXT: s_mov_b32 m0, s0
+; GFX9-NEXT: s_lshl_b32 s2, s2, 4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s0, s11, 16
-; GFX9-NEXT: v_mov_b32_e32 v1, s11
-; GFX9-NEXT: s_cmp_eq_u32 s4, 7
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 4
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s10, 16
-; GFX9-NEXT: v_perm_b32 v4, v2, v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s10
-; GFX9-NEXT: s_cmp_eq_u32 s4, 5
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s9, 16
-; GFX9-NEXT: v_perm_b32 v3, v2, v1, s0
+; GFX9-NEXT: s_movrels_b32 s1, s8
+; GFX9-NEXT: s_lshl_b32 s3, 0xffff, s2
+; GFX9-NEXT: s_andn2_b32 s1, s1, s3
+; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_lshl_or_b32 v8, v0, s2, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
; GFX9-NEXT: v_mov_b32_e32 v1, s9
-; GFX9-NEXT: s_cmp_eq_u32 s4, 3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s8, 16
-; GFX9-NEXT: v_perm_b32 v2, v2, v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NEXT: s_cmp_eq_u32 s4, 1
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 14
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s15, 16
-; GFX9-NEXT: v_perm_b32 v1, v5, v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v5, s15
-; GFX9-NEXT: s_cmp_eq_u32 s4, 15
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 12
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s14, 16
-; GFX9-NEXT: v_perm_b32 v8, v6, v5, s0
-; GFX9-NEXT: v_mov_b32_e32 v5, s14
-; GFX9-NEXT: s_cmp_eq_u32 s4, 13
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 10
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s13, 16
-; GFX9-NEXT: v_perm_b32 v7, v6, v5, s0
+; GFX9-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-NEXT: v_mov_b32_e32 v4, s12
; GFX9-NEXT: v_mov_b32_e32 v5, s13
-; GFX9-NEXT: s_cmp_eq_u32 s4, 11
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s4, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s12, 16
-; GFX9-NEXT: v_perm_b32 v6, v6, v5, s0
-; GFX9-NEXT: v_mov_b32_e32 v5, s12
-; GFX9-NEXT: s_cmp_eq_u32 s4, 9
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v9, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX9-NEXT: v_perm_b32 v5, v0, v5, s0
-; GFX9-NEXT: v_mov_b32_e32 v9, 16
-; GFX9-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-NEXT: global_store_dwordx4 v[9:10], v[5:8], off
-; GFX9-NEXT: global_store_dwordx4 v[11:12], v[1:4], off
+; GFX9-NEXT: v_mov_b32_e32 v6, s14
+; GFX9-NEXT: v_mov_b32_e32 v7, s15
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v8
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-NEXT: v_mov_b32_e32 v10, 16
+; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX9-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i16_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX8-NEXT: s_cmp_eq_u32 s4, 6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v10, 0
-; GFX8-NEXT: v_mov_b32_e32 v11, 0
+; GFX8-NEXT: s_and_b32 s1, s4, 1
+; GFX8-NEXT: s_lshr_b32 m0, s4, 1
+; GFX8-NEXT: s_lshl_b32 s1, s1, 4
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s0, s11, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: s_cmp_eq_u32 s4, 7
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 4
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s10, 16
-; GFX8-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: s_cmp_eq_u32 s4, 5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 2
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s9, 16
-; GFX8-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: s_movrels_b32 s0, s8
+; GFX8-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: s_andn2_b32 s0, s0, s1
+; GFX8-NEXT: v_or_b32_e32 v8, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: s_cmp_eq_u32 s4, 3
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s8, 16
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: s_cmp_eq_u32 s4, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 14
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s15, 16
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s15
-; GFX8-NEXT: s_cmp_eq_u32 s4, 15
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 12
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s14, 16
-; GFX8-NEXT: v_or_b32_sdwa v8, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s14
-; GFX8-NEXT: s_cmp_eq_u32 s4, 13
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 10
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s13, 16
-; GFX8-NEXT: v_or_b32_sdwa v7, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, s10
+; GFX8-NEXT: v_mov_b32_e32 v3, s11
+; GFX8-NEXT: v_mov_b32_e32 v4, s12
; GFX8-NEXT: v_mov_b32_e32 v5, s13
-; GFX8-NEXT: s_cmp_eq_u32 s4, 11
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 8
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s12, 16
-; GFX8-NEXT: v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s12
-; GFX8-NEXT: s_cmp_eq_u32 s4, 9
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_mov_b32_e32 v9, s0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v9, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v5, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v9, 16
-; GFX8-NEXT: v_mov_b32_e32 v12, 0
-; GFX8-NEXT: flat_store_dwordx4 v[9:10], v[5:8]
-; GFX8-NEXT: flat_store_dwordx4 v[11:12], v[1:4]
+; GFX8-NEXT: v_mov_b32_e32 v6, s14
+; GFX8-NEXT: v_mov_b32_e32 v7, s15
+; GFX8-NEXT: v_movreld_b32_e32 v0, v8
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
+; GFX8-NEXT: v_mov_b32_e32 v10, 16
+; GFX8-NEXT: v_mov_b32_e32 v11, 0
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i16_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX7-NEXT: s_cmp_eq_u32 s4, 6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_and_b32 s1, s4, 1
+; GFX7-NEXT: s_lshr_b32 m0, s4, 1
+; GFX7-NEXT: s_lshl_b32 s1, s1, 4
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s0, s11, 16
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_cmp_eq_u32 s4, 7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 4
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s10, 16
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: s_cmp_eq_u32 s4, 5
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 2
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s9, 16
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
+; GFX7-NEXT: s_movrels_b32 s0, s8
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX7-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX7-NEXT: s_andn2_b32 s0, s0, s1
+; GFX7-NEXT: v_or_b32_e32 v8, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s8
; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: s_cmp_eq_u32 s4, 3
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s8, 16
-; GFX7-NEXT: v_or_b32_e32 v2, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: s_cmp_eq_u32 s4, 1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 14
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s15, 16
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s15
-; GFX7-NEXT: s_cmp_eq_u32 s4, 15
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 12
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s14, 16
-; GFX7-NEXT: v_or_b32_e32 v8, v5, v6
-; GFX7-NEXT: v_mov_b32_e32 v5, s14
-; GFX7-NEXT: s_cmp_eq_u32 s4, 13
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 10
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s13, 16
-; GFX7-NEXT: v_or_b32_e32 v7, v5, v6
+; GFX7-NEXT: v_mov_b32_e32 v2, s10
+; GFX7-NEXT: v_mov_b32_e32 v3, s11
+; GFX7-NEXT: v_mov_b32_e32 v4, s12
; GFX7-NEXT: v_mov_b32_e32 v5, s13
-; GFX7-NEXT: s_cmp_eq_u32 s4, 11
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: s_cmp_eq_u32 s4, 8
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s12, 16
-; GFX7-NEXT: v_or_b32_e32 v6, v5, v6
-; GFX7-NEXT: v_mov_b32_e32 v5, s12
-; GFX7-NEXT: s_cmp_eq_u32 s4, 9
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v9, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v9, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v0
-; GFX7-NEXT: s_mov_b64 s[0:1], 16
-; GFX7-NEXT: buffer_store_dwordx4 v[5:8], off, s[0:3], 0
+; GFX7-NEXT: v_mov_b32_e32 v6, s14
+; GFX7-NEXT: v_mov_b32_e32 v7, s15
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_movreld_b32_e32 v0, v8
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i16_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 6
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s0, s11, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 7
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s11, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 4
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s0, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s0, s10, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 5
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s10, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 2
-; GFX10-NEXT: v_perm_b32 v3, v2, v1, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s0, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s0, s9, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 3
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s9, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: v_perm_b32 v2, v1, v4, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s0, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s1, s8, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 1
-; GFX10-NEXT: v_cndmask_b32_e32 v12, s8, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 14
-; GFX10-NEXT: v_perm_b32 v1, v1, v5, 0x5040100
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s15, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 15
-; GFX10-NEXT: v_cndmask_b32_e64 v4, s15, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 12
-; GFX10-NEXT: v_cndmask_b32_e64 v5, s2, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s14, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 13
-; GFX10-NEXT: v_cndmask_b32_e64 v6, s14, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 10
-; GFX10-NEXT: v_cndmask_b32_e64 v8, s2, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s13, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 11
-; GFX10-NEXT: v_cndmask_b32_e64 v9, s13, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 8
-; GFX10-NEXT: v_cndmask_b32_e64 v10, s2, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s12, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 9
-; GFX10-NEXT: v_cndmask_b32_e64 v11, s12, v0, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: v_perm_b32 v7, v5, v4, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v13, s2, v0, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s1, v0, vcc_lo
-; GFX10-NEXT: v_perm_b32 v6, v8, v6, 0x5040100
-; GFX10-NEXT: v_perm_b32 v5, v10, v9, 0x5040100
-; GFX10-NEXT: v_mov_b32_e32 v8, 16
-; GFX10-NEXT: v_perm_b32 v4, v13, v11, 0x5040100
+; GFX10-NEXT: s_and_b32 s0, s4, 1
+; GFX10-NEXT: s_lshr_b32 m0, s4, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
+; GFX10-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX10-NEXT: s_lshl_b32 s1, 0xffff, s0
; GFX10-NEXT: v_mov_b32_e32 v9, 0
-; GFX10-NEXT: v_mov_b32_e32 v10, 0
+; GFX10-NEXT: v_mov_b32_e32 v10, 16
; GFX10-NEXT: v_mov_b32_e32 v11, 0
-; GFX10-NEXT: v_perm_b32 v0, v0, v12, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[8:9], v[4:7], off
-; GFX10-NEXT: global_store_dwordx4 v[10:11], v[0:3], off
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_movrels_b32 s2, s8
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: s_andn2_b32 s1, s2, s1
+; GFX10-NEXT: v_mov_b32_e32 v1, s9
+; GFX10-NEXT: v_lshl_or_b32 v12, v8, s0, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s10
+; GFX10-NEXT: v_mov_b32_e32 v3, s11
+; GFX10-NEXT: v_mov_b32_e32 v4, s12
+; GFX10-NEXT: v_mov_b32_e32 v5, s13
+; GFX10-NEXT: v_mov_b32_e32 v6, s14
+; GFX10-NEXT: v_mov_b32_e32 v7, s15
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v12
+; GFX10-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i16_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[8:15], s[2:3], 0x0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 6
-; GFX11-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: v_dual_mov_b32 v8, 16 :: v_dual_mov_b32 v9, 0
-; GFX11-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0
+; GFX11-NEXT: s_and_b32 s0, s4, 1
+; GFX11-NEXT: s_lshr_b32 m0, s4, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX11-NEXT: s_lshl_b32 s1, 0xffff, s0
+; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v9, 0
+; GFX11-NEXT: v_mov_b32_e32 v11, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s1, s11, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 7
-; GFX11-NEXT: v_cndmask_b16 v3.l, s11, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 4
-; GFX11-NEXT: v_cndmask_b16 v3.h, s1, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_lshr_b32 s5, s10, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 5
-; GFX11-NEXT: v_cndmask_b16 v2.l, s10, v0.h, s3
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 2
-; GFX11-NEXT: v_cndmask_b16 v2.h, s5, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_lshr_b32 s2, s9, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-NEXT: v_cndmask_b16 v1.l, s9, v0.h, s1
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-NEXT: v_cndmask_b16 v1.h, s2, v0.h, s3
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_lshr_b32 s5, s8, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 1
-; GFX11-NEXT: v_cndmask_b16 v0.l, s8, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 14
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_lshr_b32 s3, s15, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 15
-; GFX11-NEXT: v_cndmask_b16 v7.l, s15, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 12
-; GFX11-NEXT: v_cndmask_b16 v7.h, s3, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_lshr_b32 s6, s14, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 13
-; GFX11-NEXT: v_cndmask_b16 v6.l, s14, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 10
-; GFX11-NEXT: v_cndmask_b16 v6.h, s6, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_lshr_b32 s3, s13, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 11
-; GFX11-NEXT: v_cndmask_b16 v5.l, s13, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s4, 8
-; GFX11-NEXT: v_cndmask_b16 v5.h, s3, v0.h, s0
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_lshr_b32 s6, s12, 16
-; GFX11-NEXT: s_cmp_eq_u32 s4, 9
-; GFX11-NEXT: v_cndmask_b16 v4.l, s12, v0.h, s2
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b16 v4.h, s6, v0.h, s0
-; GFX11-NEXT: v_cndmask_b16 v0.h, s5, v0.h, s1
+; GFX11-NEXT: s_movrels_b32 s2, s8
+; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v3, s11
+; GFX11-NEXT: s_and_not1_b32 s1, s2, s1
+; GFX11-NEXT: v_mov_b32_e32 v1, s9
+; GFX11-NEXT: v_lshl_or_b32 v12, v8, s0, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s10 :: v_dual_mov_b32 v5, s13
+; GFX11-NEXT: v_dual_mov_b32 v4, s12 :: v_dual_mov_b32 v7, s15
+; GFX11-NEXT: v_mov_b32_e32 v6, s14
+; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v12
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[8:9], v[4:7], off
-; GFX11-NEXT: global_store_b128 v[10:11], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[10:11], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -4233,369 +3158,260 @@ define amdgpu_ps void @insertelement_s_v16i16_s_v(ptr addrspace(4) inreg %ptr, i
; GFX9-LABEL: insertelement_s_v16i16_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX9-NEXT: v_mov_b32_e32 v5, s4
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GFX9-NEXT: v_mov_b32_e32 v11, 0
-; GFX9-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-NEXT: v_lshrrev_b32_e32 v9, 1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v9
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v9
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s11
-; GFX9-NEXT: s_lshr_b32 s0, s11, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: s_mov_b32 s0, 0x5040100
-; GFX9-NEXT: v_perm_b32 v4, v2, v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s10
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX9-NEXT: s_lshr_b32 s1, s10, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: v_perm_b32 v3, v2, v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s9
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX9-NEXT: s_lshr_b32 s1, s9, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX9-NEXT: v_perm_b32 v2, v2, v1, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_lshr_b32 s1, s8, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX9-NEXT: v_perm_b32 v1, v6, v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v6, s15
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GFX9-NEXT: s_lshr_b32 s1, s15, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v7, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v5, vcc
-; GFX9-NEXT: v_perm_b32 v8, v7, v6, s0
-; GFX9-NEXT: v_mov_b32_e32 v6, s14
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
-; GFX9-NEXT: s_lshr_b32 s1, s14, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v7, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v5, vcc
-; GFX9-NEXT: v_perm_b32 v7, v7, v6, s0
+; GFX9-NEXT: v_mov_b32_e32 v2, s9
+; GFX9-NEXT: v_mov_b32_e32 v3, s10
+; GFX9-NEXT: v_cndmask_b32_e32 v10, v1, v2, vcc
+; GFX9-NEXT: v_mov_b32_e32 v4, s11
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v3, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v5, s12
; GFX9-NEXT: v_mov_b32_e32 v6, s13
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GFX9-NEXT: s_lshr_b32 s1, s13, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v9, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
-; GFX9-NEXT: v_perm_b32 v6, v9, v6, s0
-; GFX9-NEXT: v_mov_b32_e32 v9, s12
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GFX9-NEXT: s_lshr_b32 s1, s12, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
-; GFX9-NEXT: v_mov_b32_e32 v10, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v5, vcc
-; GFX9-NEXT: v_perm_b32 v5, v0, v9, s0
-; GFX9-NEXT: v_mov_b32_e32 v9, 16
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: global_store_dwordx4 v[9:10], v[5:8], off
-; GFX9-NEXT: global_store_dwordx4 v[11:12], v[1:4], off
+; GFX9-NEXT: v_mov_b32_e32 v7, s14
+; GFX9-NEXT: v_mov_b32_e32 v8, s15
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v4, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 4, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v5, s[14:15]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v9
+; GFX9-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[6:7]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v9
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX9-NEXT: v_mov_b32_e32 v12, 0xffff
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v7, s[8:9]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v9
+; GFX9-NEXT: v_lshlrev_b32_e64 v11, v0, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v12
+; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v8, s[10:11]
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_and_or_b32 v10, v10, v0, v11
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v1, v10, s[12:13]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v10, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, v10, s[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v5, v10, s[14:15]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v6, v10, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v7, v10, s[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v8, v10, s[10:11]
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-NEXT: v_mov_b32_e32 v10, 16
+; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX9-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i16_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v5, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GFX8-NEXT: v_mov_b32_e32 v10, 0
-; GFX8-NEXT: v_mov_b32_e32 v11, 0
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v9
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v9
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s11
-; GFX8-NEXT: s_lshr_b32 s0, s11, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v4, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s10
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX8-NEXT: s_lshr_b32 s0, s10, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX8-NEXT: s_lshr_b32 s0, s9, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_mov_b32_e32 v1, s8
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT: s_lshr_b32 s0, s8, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v6, s15
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GFX8-NEXT: s_lshr_b32 s0, s15, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v7, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_sdwa v8, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v6, s14
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
-; GFX8-NEXT: s_lshr_b32 s0, s14, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v7, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_sdwa v7, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, s9
+; GFX8-NEXT: v_mov_b32_e32 v3, s10
+; GFX8-NEXT: v_cndmask_b32_e32 v10, v1, v2, vcc
+; GFX8-NEXT: v_mov_b32_e32 v4, s11
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v3, s[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v5, s12
; GFX8-NEXT: v_mov_b32_e32 v6, s13
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GFX8-NEXT: s_lshr_b32 s0, s13, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX8-NEXT: v_mov_b32_e32 v9, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX8-NEXT: v_or_b32_sdwa v6, v6, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v9, s12
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GFX8-NEXT: s_lshr_b32 s0, s12, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v5, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v5, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v9, 16
-; GFX8-NEXT: v_mov_b32_e32 v12, 0
-; GFX8-NEXT: flat_store_dwordx4 v[9:10], v[5:8]
-; GFX8-NEXT: flat_store_dwordx4 v[11:12], v[1:4]
+; GFX8-NEXT: v_mov_b32_e32 v7, s14
+; GFX8-NEXT: v_mov_b32_e32 v8, s15
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v4, s[2:3]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 4, v9
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v5, s[14:15]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v9
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[6:7]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v9
+; GFX8-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v7, s[8:9]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v9
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX8-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX8-NEXT: v_mov_b32_e32 v12, 0xffff
+; GFX8-NEXT: v_cndmask_b32_e64 v10, v10, v8, s[10:11]
+; GFX8-NEXT: v_lshlrev_b32_e64 v11, v0, s4
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v12
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v10
+; GFX8-NEXT: v_or_b32_e32 v10, v0, v11
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v9
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v1, v10, s[12:13]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v3, v10, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, v10, s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v5, v10, s[14:15]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v6, v10, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v7, v10, s[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v8, v10, s[10:11]
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
+; GFX8-NEXT: v_mov_b32_e32 v10, 16
+; GFX8-NEXT: v_mov_b32_e32 v11, 0
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i16_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v5, s4
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_lshrrev_b32_e32 v9, 1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v9
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v9
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s11
-; GFX7-NEXT: s_lshr_b32 s0, s11, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s10
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX7-NEXT: s_lshr_b32 s0, s10, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v1, s9
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX7-NEXT: s_lshr_b32 s0, s9, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v1, v2
; GFX7-NEXT: v_mov_b32_e32 v1, s8
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: s_lshr_b32 s0, s8, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX7-NEXT: v_mov_b32_e32 v6, s15
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GFX7-NEXT: s_lshr_b32 s0, s15, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v8, v6, v7
-; GFX7-NEXT: v_mov_b32_e32 v6, s14
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
-; GFX7-NEXT: s_lshr_b32 s0, s14, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v7, v6, v7
+; GFX7-NEXT: v_mov_b32_e32 v2, s9
+; GFX7-NEXT: v_mov_b32_e32 v3, s10
+; GFX7-NEXT: v_cndmask_b32_e32 v10, v1, v2, vcc
+; GFX7-NEXT: v_mov_b32_e32 v4, s11
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v3, s[0:1]
+; GFX7-NEXT: v_mov_b32_e32 v5, s12
; GFX7-NEXT: v_mov_b32_e32 v6, s13
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GFX7-NEXT: s_lshr_b32 s0, s13, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v9, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v9
-; GFX7-NEXT: v_mov_b32_e32 v9, s12
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GFX7-NEXT: s_lshr_b32 s0, s12, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v5, vcc
-; GFX7-NEXT: v_mov_b32_e32 v10, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v10, v5, vcc
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v5, v9, v0
-; GFX7-NEXT: s_mov_b64 s[0:1], 16
-; GFX7-NEXT: buffer_store_dwordx4 v[5:8], off, s[0:3], 0
+; GFX7-NEXT: v_mov_b32_e32 v7, s14
+; GFX7-NEXT: v_mov_b32_e32 v8, s15
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v4, s[2:3]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[14:15], 4, v9
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v5, s[14:15]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v9
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v6, s[6:7]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v9
+; GFX7-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v7, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v9
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX7-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX7-NEXT: v_cndmask_b32_e64 v10, v10, v8, s[10:11]
+; GFX7-NEXT: v_lshl_b32_e32 v11, s4, v0
+; GFX7-NEXT: v_lshl_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, v10
+; GFX7-NEXT: v_or_b32_e32 v10, v0, v11
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v9
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v1, v10, s[12:13]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v10, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v3, v10, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v4, v10, s[2:3]
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_cndmask_b32_e64 v4, v5, v10, s[14:15]
+; GFX7-NEXT: v_cndmask_b32_e64 v5, v6, v10, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v7, v10, s[8:9]
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v8, v10, s[10:11]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i16_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v13, 1, v0
+; GFX10-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX10-NEXT: s_and_b32 s5, s4, 0xffff
+; GFX10-NEXT: v_mov_b32_e32 v11, 16
+; GFX10-NEXT: v_mov_b32_e32 v12, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v13
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v13
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v13
+; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v13
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v13
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v13
+; GFX10-NEXT: v_lshlrev_b32_e64 v10, v0, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v0, v0, s5
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v13
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e32 v1, s11, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT: s_lshr_b32 s0, s11, 16
-; GFX10-NEXT: s_lshr_b32 s1, s10, 16
-; GFX10-NEXT: s_lshr_b32 s2, s9, 16
-; GFX10-NEXT: s_lshr_b32 s3, s8, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s10, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s9, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v12, s8, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT: s_lshr_b32 s0, s15, 16
-; GFX10-NEXT: v_perm_b32 v3, v3, v1, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s1, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT: v_perm_b32 v2, v6, v2, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s2, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX10-NEXT: v_perm_b32 v1, v7, v5, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s15, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v0
-; GFX10-NEXT: s_lshr_b32 s0, s14, 16
-; GFX10-NEXT: v_perm_b32 v7, v5, v8, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s14, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v0
-; GFX10-NEXT: v_mov_b32_e32 v8, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v0
-; GFX10-NEXT: s_lshr_b32 s0, s13, 16
-; GFX10-NEXT: v_perm_b32 v6, v9, v6, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v10, s13, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v0
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
+; GFX10-NEXT: v_mov_b32_e32 v2, s9
+; GFX10-NEXT: v_mov_b32_e32 v3, s10
+; GFX10-NEXT: v_mov_b32_e32 v4, s11
+; GFX10-NEXT: v_mov_b32_e32 v5, s12
+; GFX10-NEXT: v_mov_b32_e32 v6, s13
+; GFX10-NEXT: v_cndmask_b32_e32 v9, v1, v2, vcc_lo
+; GFX10-NEXT: v_mov_b32_e32 v7, s14
+; GFX10-NEXT: v_mov_b32_e32 v8, s15
+; GFX10-NEXT: v_not_b32_e32 v10, v10
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v4, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v5, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v6, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v7, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v8, s5
+; GFX10-NEXT: v_and_or_b32 v14, v9, v10, v0
; GFX10-NEXT: v_mov_b32_e32 v9, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v11, s0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v0
-; GFX10-NEXT: s_lshr_b32 s0, s12, 16
-; GFX10-NEXT: v_perm_b32 v5, v11, v10, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v13, s12, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v0
; GFX10-NEXT: v_mov_b32_e32 v10, 0
-; GFX10-NEXT: v_mov_b32_e32 v11, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v14, s0, v4, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s3, v4, vcc_lo
-; GFX10-NEXT: v_perm_b32 v4, v14, v13, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v0, v12, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[8:9], v[4:7], off
-; GFX10-NEXT: global_store_dwordx4 v[10:11], v[0:3], off
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v14, s6
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v14, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, v14, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, v14, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v5, v14, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v6, v14, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v7, v14, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v8, v14, s5
+; GFX10-NEXT: global_store_dwordx4 v[9:10], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[11:12], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i16_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[8:15], s[2:3], 0x0
-; GFX11-NEXT: v_mov_b16_e32 v1.h, s4
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 2, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 5, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 3, v0
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 0, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 1, v0
-; GFX11-NEXT: v_dual_mov_b32 v9, 16 :: v_dual_mov_b32 v10, 0
-; GFX11-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 0
+; GFX11-NEXT: v_lshrrev_b32_e32 v13, 1, v0
+; GFX11-NEXT: s_and_b32 s5, s4, 0xffff
+; GFX11-NEXT: v_dual_mov_b32 v11, 16 :: v_dual_and_b32 v0, 1, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v13
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v13
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v13
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v13
+; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v13
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v13
+; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v10, v0, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v0, v0, s5
+; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v13
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s7, s11, 16
-; GFX11-NEXT: v_cndmask_b16 v3.l, s10, v1.h, s1
-; GFX11-NEXT: s_lshr_b32 s1, s10, 16
-; GFX11-NEXT: v_cndmask_b16 v4.h, s7, v1.h, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 15, v0
-; GFX11-NEXT: v_cndmask_b16 v2.l, s9, v1.h, s3
-; GFX11-NEXT: s_lshr_b32 s3, s9, 16
-; GFX11-NEXT: v_cndmask_b16 v3.h, s1, v1.h, s2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 13, v0
-; GFX11-NEXT: v_cndmask_b16 v2.h, s3, v1.h, s4
-; GFX11-NEXT: s_lshr_b32 s3, s15, 16
-; GFX11-NEXT: v_cndmask_b16 v4.l, s11, v1.h, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 12, v0
-; GFX11-NEXT: v_cndmask_b16 v8.h, s3, v1.h, s0
-; GFX11-NEXT: s_lshr_b32 s0, s14, 16
-; GFX11-NEXT: s_lshr_b32 s3, s13, 16
-; GFX11-NEXT: v_cndmask_b16 v7.h, s0, v1.h, s2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 11, v0
-; GFX11-NEXT: v_cndmask_b16 v8.l, s15, v1.h, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v7.l, s14, v1.h, s1
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 8, v0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 9, v0
-; GFX11-NEXT: v_cndmask_b16 v6.h, s3, v1.h, s0
-; GFX11-NEXT: s_lshr_b32 s0, s12, 16
-; GFX11-NEXT: v_cndmask_b16 v1.l, s8, v1.h, s5
-; GFX11-NEXT: s_lshr_b32 s5, s8, 16
-; GFX11-NEXT: v_cndmask_b16 v6.l, s13, v1.h, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v5.l, s12, v1.h, s1
-; GFX11-NEXT: v_cndmask_b16 v5.h, s0, v1.h, s2
-; GFX11-NEXT: v_cndmask_b16 v1.h, s5, v1.h, s6
+; GFX11-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v2, s9
+; GFX11-NEXT: v_dual_mov_b32 v3, s10 :: v_dual_mov_b32 v4, s11
+; GFX11-NEXT: v_dual_mov_b32 v5, s12 :: v_dual_mov_b32 v6, s13
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v9, v1, v2, vcc_lo
+; GFX11-NEXT: v_dual_mov_b32 v7, s14 :: v_dual_mov_b32 v8, s15
+; GFX11-NEXT: v_not_b32_e32 v10, v10
+; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v3, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v4, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v5, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v6, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v7, s4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v9, v9, v8, s5
+; GFX11-NEXT: v_and_or_b32 v14, v9, v10, v0
+; GFX11-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
+; GFX11-NEXT: v_mov_b32_e32 v12, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v14, s6
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v14, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v14, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, v14, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v5, v14, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v6, v14, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v7, v14, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v8, v14, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[9:10], v[5:8], off
-; GFX11-NEXT: global_store_b128 v[11:12], v[1:4], off
+; GFX11-NEXT: global_store_b128 v[9:10], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[11:12], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -4607,364 +3423,258 @@ define amdgpu_ps void @insertelement_s_v16i16_v_v(ptr addrspace(4) inreg %ptr, i
; GFX9-LABEL: insertelement_s_v16i16_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: v_lshrrev_b32_e32 v10, 1, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
+; GFX9-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v10
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: s_mov_b32 s3, 0x5040100
-; GFX9-NEXT: v_perm_b32 v5, v3, v2, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_perm_b32 v4, v3, v2, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_perm_b32 v3, v3, v2, s3
; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_perm_b32 v2, v6, v2, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s7
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 14, v1
-; GFX9-NEXT: s_lshr_b32 s0, s7, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v7, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX9-NEXT: v_perm_b32 v9, v7, v6, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s6
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v1
-; GFX9-NEXT: s_lshr_b32 s0, s6, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v7, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 13, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX9-NEXT: v_perm_b32 v8, v7, v6, s3
-; GFX9-NEXT: v_mov_b32_e32 v6, s5
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 10, v1
-; GFX9-NEXT: s_lshr_b32 s0, s5, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v7, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 11, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX9-NEXT: v_perm_b32 v7, v7, v6, s3
+; GFX9-NEXT: v_mov_b32_e32 v9, s7
+; GFX9-NEXT: v_mov_b32_e32 v8, s6
+; GFX9-NEXT: v_mov_b32_e32 v7, s5
; GFX9-NEXT: v_mov_b32_e32 v6, s4
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v1
-; GFX9-NEXT: s_lshr_b32 s0, s4, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v10, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 9, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
-; GFX9-NEXT: v_perm_b32 v6, v0, v6, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, 16
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v10, 0
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[6:9], off
-; GFX9-NEXT: global_store_dwordx4 v[10:11], v[2:5], off
+; GFX9-NEXT: v_mov_b32_e32 v5, s3
+; GFX9-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v11, v2, v3, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v4, s[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v5, s[2:3]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v6, s[4:5]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v7, s[6:7]
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX9-NEXT: v_mov_b32_e32 v12, 0xffff
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v8, s[8:9]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v10
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, v1, v12
+; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v9, s[10:11]
+; GFX9-NEXT: v_not_b32_e32 v1, v1
+; GFX9-NEXT: v_and_or_b32 v11, v11, v1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v10
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v11, s[12:13]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v11, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, v11, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v11, s[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v6, v11, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v11, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v11, s[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, v11, s[10:11]
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-NEXT: v_mov_b32_e32 v10, 16
+; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
+; GFX9-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i16_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX8-NEXT: v_mov_b32_e32 v10, 0
-; GFX8-NEXT: v_mov_b32_e32 v11, 0
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 1, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v10
+; GFX8-NEXT: v_and_b32_e32 v1, 1, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_or_b32_sdwa v5, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v6, s7
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 14, v1
-; GFX8-NEXT: s_lshr_b32 s0, s7, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v7, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 15, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_sdwa v9, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v1
-; GFX8-NEXT: s_lshr_b32 s0, s6, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v7, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_sdwa v8, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v6, s5
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 10, v1
-; GFX8-NEXT: s_lshr_b32 s0, s5, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v7, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 11, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX8-NEXT: v_or_b32_sdwa v7, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v9, s7
+; GFX8-NEXT: v_mov_b32_e32 v8, s6
+; GFX8-NEXT: v_mov_b32_e32 v7, s5
; GFX8-NEXT: v_mov_b32_e32 v6, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v1
-; GFX8-NEXT: s_lshr_b32 s0, s4, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 9, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v6, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[6:9]
-; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_cndmask_b32_e32 v11, v2, v3, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v4, s[0:1]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v5, s[2:3]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v6, s[4:5]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v7, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v8, s[8:9]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v10
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX8-NEXT: v_mov_b32_e32 v12, 0xffff
+; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v9, s[10:11]
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v1, v12
+; GFX8-NEXT: v_bfi_b32 v1, v1, 0, v11
+; GFX8-NEXT: v_or_b32_e32 v11, v1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v10
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v11, s[12:13]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v11, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, v11, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v11, s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v6, v11, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v7, v11, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v11, s[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v9, v11, s[10:11]
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
+; GFX8-NEXT: v_mov_b32_e32 v10, 16
+; GFX8-NEXT: v_mov_b32_e32 v11, 0
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i16_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v10, 1, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v10
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: s_lshr_b32 s3, s3, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v5, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX7-NEXT: s_lshr_b32 s2, s2, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v2, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX7-NEXT: s_lshr_b32 s1, s1, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; GFX7-NEXT: v_mov_b32_e32 v3, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v3, v2, v3
; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT: s_lshr_b32 s0, s0, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_mov_b32_e32 v6, s7
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v1
-; GFX7-NEXT: s_lshr_b32 s0, s7, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v9, v6, v7
-; GFX7-NEXT: v_mov_b32_e32 v6, s6
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v1
-; GFX7-NEXT: s_lshr_b32 s0, s6, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v8, v6, v7
-; GFX7-NEXT: v_mov_b32_e32 v6, s5
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v1
-; GFX7-NEXT: s_lshr_b32 s0, s5, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v7, v6, v7
+; GFX7-NEXT: v_mov_b32_e32 v9, s7
+; GFX7-NEXT: v_mov_b32_e32 v8, s6
+; GFX7-NEXT: v_mov_b32_e32 v7, s5
; GFX7-NEXT: v_mov_b32_e32 v6, s4
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v1
-; GFX7-NEXT: s_lshr_b32 s0, s4, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v10, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v10, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v0
-; GFX7-NEXT: s_mov_b64 s[0:1], 16
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0
+; GFX7-NEXT: v_mov_b32_e32 v5, s3
+; GFX7-NEXT: v_mov_b32_e32 v4, s2
+; GFX7-NEXT: v_cndmask_b32_e32 v11, v2, v3, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v10
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v4, s[0:1]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v10
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v5, s[2:3]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v10
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v6, s[4:5]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v10
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v7, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v8, s[8:9]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v10
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v9, s[10:11]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v11
+; GFX7-NEXT: v_or_b32_e32 v11, v1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v10
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, v11, s[12:13]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v11, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, v11, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v11, s[2:3]
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: buffer_store_dwordx4 v[2:5], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_cndmask_b32_e64 v4, v6, v11, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e64 v5, v7, v11, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v8, v11, s[8:9]
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v9, v11, s[10:11]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b64 s[0:1], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i16_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx8 s[0:7], s[2:3], 0x0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v14, 1, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX10-NEXT: v_mov_b32_e32 v12, 16
+; GFX10-NEXT: v_mov_b32_e32 v13, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v11, v1, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v1, v11
+; GFX10-NEXT: v_mov_b32_e32 v11, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s3, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v1
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s2, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v1
-; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v13, s0, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v1
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s3, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v1
-; GFX10-NEXT: v_perm_b32 v4, v4, v2, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s2, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v1
-; GFX10-NEXT: v_perm_b32 v3, v6, v3, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v1
-; GFX10-NEXT: s_lshr_b32 s1, s7, 16
-; GFX10-NEXT: v_perm_b32 v2, v7, v5, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s7, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v1
-; GFX10-NEXT: s_lshr_b32 s1, s6, 16
-; GFX10-NEXT: v_perm_b32 v8, v5, v8, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s6, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v1
-; GFX10-NEXT: s_lshr_b32 s1, s5, 16
-; GFX10-NEXT: v_perm_b32 v7, v7, v6, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s5, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v1
-; GFX10-NEXT: v_cndmask_b32_e32 v10, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v1
-; GFX10-NEXT: s_lshr_b32 s1, s4, 16
-; GFX10-NEXT: v_perm_b32 v6, v10, v9, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v11, s4, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v1
-; GFX10-NEXT: v_mov_b32_e32 v9, 16
+; GFX10-NEXT: v_mov_b32_e32 v3, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
+; GFX10-NEXT: v_mov_b32_e32 v9, s7
+; GFX10-NEXT: v_mov_b32_e32 v8, s6
+; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v6, s4
+; GFX10-NEXT: v_mov_b32_e32 v5, s3
+; GFX10-NEXT: v_mov_b32_e32 v4, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v10, v2, v3, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v14
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v14
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v14
+; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v14
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v14
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v4, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v14
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v14
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v5, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v7, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v9, s5
+; GFX10-NEXT: v_and_or_b32 v15, v10, v1, v0
; GFX10-NEXT: v_mov_b32_e32 v10, 0
-; GFX10-NEXT: v_cndmask_b32_e32 v12, s1, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX10-NEXT: v_perm_b32 v5, v12, v11, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s0, v0, vcc_lo
-; GFX10-NEXT: v_mov_b32_e32 v11, 0
-; GFX10-NEXT: v_mov_b32_e32 v12, 0
-; GFX10-NEXT: v_perm_b32 v1, v0, v13, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[9:10], v[5:8], off
-; GFX10-NEXT: global_store_dwordx4 v[11:12], v[1:4], off
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v2, v15, s6
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v15, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v4, v15, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v5, v15, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v6, v15, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v7, v15, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v8, v15, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v9, v15, s5
+; GFX10-NEXT: global_store_dwordx4 v[10:11], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i16_v_v:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b256 s[8:15], s[2:3], 0x0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 2, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 5, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 3, v1
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 0, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 1, v1
-; GFX11-NEXT: v_dual_mov_b32 v10, 16 :: v_dual_mov_b32 v11, 0
-; GFX11-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 0
+; GFX11-NEXT: s_load_b256 s[0:7], s[2:3], 0x0
+; GFX11-NEXT: v_lshrrev_b32_e32 v14, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX11-NEXT: v_and_b32_e32 v1, 1, v1
+; GFX11-NEXT: v_mov_b32_e32 v13, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v14
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e64 v11, v1, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v1, v0
+; GFX11-NEXT: v_not_b32_e32 v1, v11
+; GFX11-NEXT: v_mov_b32_e32 v11, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s7, s11, 16
-; GFX11-NEXT: v_cndmask_b16 v4.l, s10, v0.l, s1
-; GFX11-NEXT: s_lshr_b32 s1, s10, 16
-; GFX11-NEXT: v_cndmask_b16 v5.h, s7, v0.l, s0
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 15, v1
-; GFX11-NEXT: v_cndmask_b16 v3.l, s9, v0.l, s3
-; GFX11-NEXT: s_lshr_b32 s3, s9, 16
-; GFX11-NEXT: v_cndmask_b16 v4.h, s1, v0.l, s2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 13, v1
-; GFX11-NEXT: v_cndmask_b16 v3.h, s3, v0.l, s4
-; GFX11-NEXT: s_lshr_b32 s3, s15, 16
-; GFX11-NEXT: v_cndmask_b16 v5.l, s11, v0.l, vcc_lo
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 12, v1
-; GFX11-NEXT: v_cndmask_b16 v9.h, s3, v0.l, s0
-; GFX11-NEXT: s_lshr_b32 s0, s14, 16
-; GFX11-NEXT: s_lshr_b32 s3, s13, 16
-; GFX11-NEXT: v_cndmask_b16 v8.h, s0, v0.l, s2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 11, v1
-; GFX11-NEXT: v_cndmask_b16 v9.l, s15, v0.l, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v8.l, s14, v0.l, s1
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 8, v1
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 9, v1
-; GFX11-NEXT: v_cndmask_b16 v7.h, s3, v0.l, s0
-; GFX11-NEXT: s_lshr_b32 s0, s12, 16
-; GFX11-NEXT: v_cndmask_b16 v2.l, s8, v0.l, s5
-; GFX11-NEXT: s_lshr_b32 s5, s8, 16
-; GFX11-NEXT: v_cndmask_b16 v7.l, s13, v0.l, vcc_lo
-; GFX11-NEXT: v_cndmask_b16 v6.l, s12, v0.l, s1
-; GFX11-NEXT: v_cndmask_b16 v6.h, s0, v0.l, s2
-; GFX11-NEXT: v_cndmask_b16 v2.h, s5, v0.l, s6
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v10, v2, v3, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v14
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v14
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v14
+; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v14
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v14
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v4, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v14
+; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v14
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v5, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v6, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v7, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v8, s4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v10, v10, v9, s5
+; GFX11-NEXT: v_and_or_b32 v15, v10, v1, v0
+; GFX11-NEXT: v_mov_b32_e32 v10, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_cndmask_b32 v1, v3, v15
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v2, v15, s6
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v4, v15, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v5, v15, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v6, v15, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v7, v15, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v8, v15, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v9, v15, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[10:11], v[6:9], off
-; GFX11-NEXT: global_store_b128 v[12:13], v[2:5], off
+; GFX11-NEXT: global_store_b128 v[10:11], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[12:13], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -4977,207 +3687,141 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
; GFX9-NEXT: global_load_dwordx4 v[7:10], v[0:1], off offset:16
-; GFX9-NEXT: v_mov_b32_e32 v13, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2
-; GFX9-NEXT: s_mov_b32 s8, 0x5040100
-; GFX9-NEXT: v_mov_b32_e32 v0, 16
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v15, v2, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v1
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v1
; GFX9-NEXT: v_mov_b32_e32 v11, 0
; GFX9-NEXT: v_mov_b32_e32 v12, 0
+; GFX9-NEXT: v_mov_b32_e32 v13, 16
+; GFX9-NEXT: v_mov_b32_e32 v14, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v6, v13, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v6, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v5, v13, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v4, v13, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v3, v13, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v5, v5, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_cndmask_b32_sdwa v4, v4, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v3, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 13, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 14, v2
-; GFX9-NEXT: v_perm_b32 v6, v6, v14, s8
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v10, v13, s[6:7]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 12, v2
-; GFX9-NEXT: v_cndmask_b32_sdwa v10, v10, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_perm_b32 v5, v5, v15, s8
-; GFX9-NEXT: v_cndmask_b32_e64 v15, v9, v13, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v9, v9, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 10, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 11, v2
-; GFX9-NEXT: v_perm_b32 v10, v10, v14, s8
-; GFX9-NEXT: v_cndmask_b32_e32 v14, v8, v13, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 9, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v7, v13, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v8, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v7, v13, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v9, v9, v15, s8
-; GFX9-NEXT: v_perm_b32 v8, v8, v14, s8
-; GFX9-NEXT: v_perm_b32 v7, v7, v2, s8
-; GFX9-NEXT: v_perm_b32 v4, v4, v16, s8
-; GFX9-NEXT: v_perm_b32 v3, v3, v17, s8
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX9-NEXT: global_store_dwordx4 v[11:12], v[3:6], off
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v7, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v9, s[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11]
+; GFX9-NEXT: v_and_or_b32 v15, v2, v0, v15
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, v15, s[12:13]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v15, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v5, v15, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v6, v15, s[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v7, v15, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, v15, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v9, v15, s[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v10, v15, s[10:11]
+; GFX9-NEXT: global_store_dwordx4 v[11:12], v[0:3], off
+; GFX9-NEXT: global_store_dwordx4 v[13:14], v[4:7], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i16_s_v:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 16, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[3:4]
+; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx4 v[7:10], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v13, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 15, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 14, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 13, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 11, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 9, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX8-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_e64 v15, v2, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v1
; GFX8-NEXT: v_mov_b32_e32 v11, 0
; GFX8-NEXT: v_mov_b32_e32 v12, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v13, 16
+; GFX8-NEXT: v_mov_b32_e32 v14, 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_cndmask_b32_e64 v14, v6, v13, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_sdwa v6, v6, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v5, v13, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 10, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v16, v4, v13, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v17, v3, v13, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_sdwa v5, v5, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_sdwa v4, v4, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v3, v3, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v2
-; GFX8-NEXT: v_or_b32_sdwa v6, v14, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e64 v14, v10, v13, s[6:7]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 4, v2
-; GFX8-NEXT: v_cndmask_b32_sdwa v10, v10, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v5, v15, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e64 v15, v9, v13, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_sdwa v9, v9, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v2
-; GFX8-NEXT: v_or_b32_sdwa v10, v14, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v14, v8, v13, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v13, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_sdwa v8, v8, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v7, v13, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v9, v15, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v8, v14, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v7, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v4, v16, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v3, v17, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[11:12], v[7:10]
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v7, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v9, s[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[10:11]
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v2
+; GFX8-NEXT: v_or_b32_e32 v15, v0, v15
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v15, s[12:13]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v15, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v15, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v15, s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v7, v15, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v8, v15, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v9, v15, s[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v10, v15, s[10:11]
+; GFX8-NEXT: flat_store_dwordx4 v[11:12], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[13:14], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v16i16_s_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: buffer_load_dwordx4 v[7:10], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v2
-; GFX7-NEXT: s_mov_b64 s[4:5], 16
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: s_mov_b32 s18, 0
+; GFX7-NEXT: s_mov_b32 s19, 0xf000
+; GFX7-NEXT: s_mov_b64 s[16:17], 0
+; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[16:19], 0 addr64
+; GFX7-NEXT: buffer_load_dwordx4 v[7:10], v[0:1], s[16:19], 0 addr64 offset:16
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 1, v2
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v2, s0, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v0
+; GFX7-NEXT: s_mov_b64 s[16:17], 0
+; GFX7-NEXT: s_mov_b32 s18, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v11, v5, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v12, v4, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v13, v3, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v2
+; GFX7-NEXT: v_cndmask_b32_e32 v11, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v5, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v6, s[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v14, v10, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v9, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v16, v8, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v17, v7, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v8, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v11, 0xffff, v11
-; GFX7-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX7-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX7-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX7-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX7-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v10
-; GFX7-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v11, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v12, v4
-; GFX7-NEXT: v_or_b32_e32 v0, v13, v6
-; GFX7-NEXT: v_or_b32_e32 v7, v14, v7
-; GFX7-NEXT: v_or_b32_e32 v6, v15, v9
-; GFX7-NEXT: v_or_b32_e32 v5, v16, v8
-; GFX7-NEXT: v_or_b32_e32 v4, v17, v10
-; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v7, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v8, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v9, s[8:9]
+; GFX7-NEXT: v_cndmask_b32_e64 v11, v11, v10, s[10:11]
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v11
+; GFX7-NEXT: v_or_b32_e32 v11, v1, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v3, v11, s[12:13]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v11, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v5, v11, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v6, v11, s[2:3]
+; GFX7-NEXT: v_cndmask_b32_e64 v4, v7, v11, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e64 v5, v8, v11, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v9, v11, s[8:9]
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v10, v11, s[10:11]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GFX7-NEXT: s_mov_b64 s[16:17], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i16_s_v:
@@ -5185,62 +3829,45 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
; GFX10-NEXT: global_load_dwordx4 v[7:10], v[0:1], off offset:16
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX10-NEXT: v_mov_b32_e32 v11, 16
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 1, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX10-NEXT: s_and_b32 s5, s2, 0xffff
; GFX10-NEXT: v_mov_b32_e32 v12, 0
-; GFX10-NEXT: v_mov_b32_e32 v13, 0
+; GFX10-NEXT: v_mov_b32_e32 v13, 16
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 4, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 5, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 6, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v0
; GFX10-NEXT: v_mov_b32_e32 v14, 0
+; GFX10-NEXT: v_lshlrev_b32_e64 v11, v2, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v2, s5
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v0
+; GFX10-NEXT: v_not_b32_e32 v11, v11
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v6, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v5, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX10-NEXT: v_cndmask_b32_e64 v15, v4, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v16, v3, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v5, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v6, s1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e64 v17, v10, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v9, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX10-NEXT: v_cndmask_b32_e64 v19, v8, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX10-NEXT: v_cndmask_b32_e64 v20, v7, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v21, v3, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v2
-; GFX10-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v4, v15, 0x5040100
-; GFX10-NEXT: v_perm_b32 v0, v21, v16, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v2
-; GFX10-NEXT: v_perm_b32 v2, v5, v6, 0x5040100
-; GFX10-NEXT: v_perm_b32 v6, v9, v18, 0x5040100
-; GFX10-NEXT: v_perm_b32 v5, v8, v19, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_e64 v22, v7, s2, vcc_lo
-; GFX10-NEXT: v_perm_b32 v7, v10, v17, 0x5040100
-; GFX10-NEXT: v_perm_b32 v4, v22, v20, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[11:12], v[4:7], off
-; GFX10-NEXT: global_store_dwordx4 v[13:14], v[0:3], off
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v7, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v8, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v10, s5
+; GFX10-NEXT: v_and_or_b32 v15, v1, v11, v2
+; GFX10-NEXT: v_mov_b32_e32 v11, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v15, s6
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v15, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v15, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v15, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v7, v15, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v8, v15, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v9, v15, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v10, v15, s5
+; GFX10-NEXT: global_store_dwordx4 v[11:12], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[13:14], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v16i16_s_v:
@@ -5248,45 +3875,50 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
; GFX11-NEXT: global_load_b128 v[7:10], v[0:1], off offset:16
-; GFX11-NEXT: v_cmp_eq_u32_e64 s8, 14, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s9, 15, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s10, 12, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s11, 13, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s12, 10, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s13, 11, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s14, 8, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s15, 9, v2
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 2, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 3, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v2
-; GFX11-NEXT: v_cmp_eq_u32_e64 s7, 1, v2
-; GFX11-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v2
+; GFX11-NEXT: s_and_b32 s5, s2, 0xffff
+; GFX11-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_mov_b32 v13, 16
+; GFX11-NEXT: v_and_b32_e32 v2, 1, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 4, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 5, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 6, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v11, v2, 0xffff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v2, s5
+; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v0
+; GFX11-NEXT: v_not_b32_e32 v11, v11
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_cndmask_b16 v6.l, v6.l, s2, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v5, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v6, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v10.l, v10.l, s2, s8
-; GFX11-NEXT: v_cndmask_b16 v10.h, v10.h, s2, s9
-; GFX11-NEXT: v_cndmask_b16 v9.l, v9.l, s2, s10
-; GFX11-NEXT: v_cndmask_b16 v9.h, v9.h, s2, s11
-; GFX11-NEXT: v_cndmask_b16 v8.l, v8.l, s2, s12
-; GFX11-NEXT: v_cndmask_b16 v8.h, v8.h, s2, s13
-; GFX11-NEXT: v_cndmask_b16 v7.l, v7.l, s2, s14
-; GFX11-NEXT: v_cndmask_b16 v7.h, v7.h, s2, s15
-; GFX11-NEXT: v_cndmask_b16 v6.h, v6.h, s2, s0
-; GFX11-NEXT: v_cndmask_b16 v5.l, v5.l, s2, s1
-; GFX11-NEXT: v_cndmask_b16 v5.h, v5.h, s2, s3
-; GFX11-NEXT: v_cndmask_b16 v4.l, v4.l, s2, s4
-; GFX11-NEXT: v_cndmask_b16 v4.h, v4.h, s2, s5
-; GFX11-NEXT: v_cndmask_b16 v3.l, v3.l, s2, s6
-; GFX11-NEXT: v_cndmask_b16 v3.h, v3.h, s2, s7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v8, s4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v10, s5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v15, v1, v11, v2
+; GFX11-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v14, 0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v15, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v3, v15, s6
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v5, v15, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v6, v15, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v7, v15, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v8, v15, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v9, v15, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v10, v15, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[0:1], v[7:10], off
-; GFX11-NEXT: global_store_b128 v[11:12], v[3:6], off
+; GFX11-NEXT: global_store_b128 v[11:12], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[13:14], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(1) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -5299,262 +3931,76 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
; GFX9-NEXT: global_load_dwordx4 v[7:10], v[0:1], off offset:16
-; GFX9-NEXT: s_cmp_eq_u32 s2, 6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 7
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 4
-; GFX9-NEXT: s_cselect_b64 s[30:31], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 5
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 2
-; GFX9-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 3
-; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 0
-; GFX9-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 1
-; GFX9-NEXT: s_cselect_b64 s[12:13], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 14
-; GFX9-NEXT: s_cselect_b64 s[14:15], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 15
-; GFX9-NEXT: s_cselect_b64 s[16:17], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 12
-; GFX9-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 13
-; GFX9-NEXT: s_cselect_b64 s[20:21], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 10
-; GFX9-NEXT: s_cselect_b64 s[22:23], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 11
-; GFX9-NEXT: s_cselect_b64 s[24:25], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 8
-; GFX9-NEXT: s_mov_b32 s3, 0x5040100
-; GFX9-NEXT: s_cselect_b64 s[26:27], -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 9
-; GFX9-NEXT: s_cselect_b64 s[28:29], -1, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, 16
+; GFX9-NEXT: s_and_b32 s1, s2, 1
+; GFX9-NEXT: s_lshl_b32 s1, s1, 4
+; GFX9-NEXT: s_lshr_b32 s0, s2, 1
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, s1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v11, 0
+; GFX9-NEXT: s_not_b32 s1, s1
+; GFX9-NEXT: v_mov_b32_e32 v11, 16
; GFX9-NEXT: v_mov_b32_e32 v12, 0
-; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cndmask_b32_e32 v13, v6, v2, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v5, v2, s[30:31]
-; GFX9-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_cndmask_b32_e64 v15, v4, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v4, v4, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_cndmask_b32_e64 v16, v3, v2, s[10:11]
-; GFX9-NEXT: v_cndmask_b32_sdwa v3, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[16:17]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e64 v17, v10, v2, s[14:15]
-; GFX9-NEXT: v_cndmask_b32_sdwa v10, v10, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[20:21]
-; GFX9-NEXT: v_perm_b32 v6, v6, v13, s3
-; GFX9-NEXT: v_cndmask_b32_e64 v13, v9, v2, s[18:19]
-; GFX9-NEXT: v_cndmask_b32_sdwa v9, v9, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[24:25]
-; GFX9-NEXT: v_perm_b32 v5, v5, v14, s3
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v8, v2, s[22:23]
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v8, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[28:29]
-; GFX9-NEXT: v_perm_b32 v9, v9, v13, s3
-; GFX9-NEXT: v_cndmask_b32_e64 v13, v7, v2, s[26:27]
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v7, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v10, v10, v17, s3
-; GFX9-NEXT: v_perm_b32 v8, v8, v14, s3
-; GFX9-NEXT: v_perm_b32 v7, v2, v13, s3
-; GFX9-NEXT: v_perm_b32 v4, v4, v15, s3
-; GFX9-NEXT: v_perm_b32 v3, v3, v16, s3
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[7:10], off
-; GFX9-NEXT: global_store_dwordx4 v[11:12], v[3:6], off
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v13, v3
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v2, v13, s1, v2
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX9-NEXT: global_store_dwordx4 v[11:12], v[7:10], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i16_v_s:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 16, v0
-; GFX8-NEXT: v_addc_u32_e32 v4, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[3:4]
-; GFX8-NEXT: flat_load_dwordx4 v[7:10], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s2, 14
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 15
-; GFX8-NEXT: v_mov_b32_e32 v11, 0
-; GFX8-NEXT: v_mov_b32_e32 v12, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, 16
+; GFX8-NEXT: v_add_u32_e32 v7, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v8, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
+; GFX8-NEXT: flat_load_dwordx4 v[7:10], v[7:8]
+; GFX8-NEXT: s_and_b32 s0, s2, 1
+; GFX8-NEXT: s_lshl_b32 s0, s0, 4
+; GFX8-NEXT: s_lshr_b32 m0, s2, 1
+; GFX8-NEXT: v_mov_b32_e32 v13, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v13, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_cndmask_b32_e32 v13, v6, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 12
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 13
-; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; GFX8-NEXT: v_cndmask_b32_e64 v5, v5, v2, s[0:1]
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 10
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 11
-; GFX8-NEXT: v_lshrrev_b32_e32 v15, 16, v4
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 8
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v2, s[4:5]
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: v_cndmask_b32_e64 v15, v15, v2, s[6:7]
-; GFX8-NEXT: s_cmp_eq_u32 s2, 9
-; GFX8-NEXT: v_lshrrev_b32_e32 v16, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 6
-; GFX8-NEXT: v_or_b32_sdwa v4, v4, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v16, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 7
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v2, s[4:5]
+; GFX8-NEXT: v_mov_b32_e32 v11, 16
+; GFX8-NEXT: v_mov_b32_e32 v12, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v17, 16, v10
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v14, v14, v2, s[0:1]
-; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 4
-; GFX8-NEXT: v_or_b32_sdwa v6, v13, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v14
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v17, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 5
-; GFX8-NEXT: v_or_b32_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v9
-; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 2
-; GFX8-NEXT: v_cndmask_b32_e32 v13, v13, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 3
-; GFX8-NEXT: v_lshrrev_b32_e32 v14, 16, v8
-; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 0
-; GFX8-NEXT: v_or_b32_sdwa v9, v9, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v13, v14, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v14, v7, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; GFX8-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; GFX8-NEXT: v_cndmask_b32_sdwa v2, v7, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v10, v10, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v8, v8, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v7, v14, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[11:12], v[7:10]
+; GFX8-NEXT: v_movrels_b32_e32 v13, v3
+; GFX8-NEXT: v_bfi_b32 v13, s0, 0, v13
+; GFX8-NEXT: v_or_b32_e32 v2, v13, v2
+; GFX8-NEXT: v_movreld_b32_e32 v3, v2
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
+; GFX8-NEXT: flat_store_dwordx4 v[11:12], v[7:10]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v16i16_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
; GFX7-NEXT: buffer_load_dwordx4 v[7:10], v[0:1], s[4:7], 0 addr64 offset:16
-; GFX7-NEXT: s_cmp_eq_u32 s2, 6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 7
-; GFX7-NEXT: s_mov_b64 s[4:5], 16
+; GFX7-NEXT: s_and_b32 s0, s2, 1
+; GFX7-NEXT: s_lshr_b32 m0, s2, 1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 4
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX7-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 4
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 2
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v16, v6, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 3
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v4
-; GFX7-NEXT: v_or_b32_e32 v6, v0, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v11, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 1
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 14
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v4
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v12, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 15
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v10
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 12
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v13, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 13
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 10
-; GFX7-NEXT: v_or_b32_e32 v10, v1, v10
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v9
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v14, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 11
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v8, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 8
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v16
-; GFX7-NEXT: v_or_b32_e32 v9, v1, v9
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v8
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v15, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 9
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT: v_or_b32_e32 v8, v1, v8
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v7, v1, v0
-; GFX7-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: v_movrels_b32_e32 v1, v3
+; GFX7-NEXT: v_bfi_b32 v1, s0, 0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_movreld_b32_e32 v3, v0
; GFX7-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0
+; GFX7-NEXT: s_mov_b64 s[4:5], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[7:10], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i16_v_s:
@@ -5562,78 +4008,22 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
; GFX10-NEXT: global_load_dwordx4 v[7:10], v[0:1], off offset:16
-; GFX10-NEXT: s_cmp_eq_u32 s2, 6
+; GFX10-NEXT: s_and_b32 s0, s2, 1
+; GFX10-NEXT: s_lshr_b32 m0, s2, 1
+; GFX10-NEXT: s_lshl_b32 s0, s0, 4
; GFX10-NEXT: v_mov_b32_e32 v11, 16
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 7
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xffff, s0
; GFX10-NEXT: v_mov_b32_e32 v12, 0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 4
-; GFX10-NEXT: v_mov_b32_e32 v13, 0
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 5
-; GFX10-NEXT: v_mov_b32_e32 v14, 0
-; GFX10-NEXT: s_cselect_b32 s3, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 2
-; GFX10-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 3
-; GFX10-NEXT: s_cselect_b32 s5, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 0
-; GFX10-NEXT: s_cselect_b32 s6, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 1
-; GFX10-NEXT: s_cselect_b32 s7, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 14
-; GFX10-NEXT: s_cselect_b32 s8, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 15
-; GFX10-NEXT: s_cselect_b32 s9, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 12
-; GFX10-NEXT: s_cselect_b32 s10, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 13
-; GFX10-NEXT: s_cselect_b32 s11, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 10
-; GFX10-NEXT: s_cselect_b32 s12, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 11
-; GFX10-NEXT: s_cselect_b32 s13, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 8
-; GFX10-NEXT: s_cselect_b32 s14, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 9
-; GFX10-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc_lo
-; GFX10-NEXT: s_mov_b32 vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_e64 v15, v4, v2, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v6, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s3
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v5, v2, s1
-; GFX10-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s5
-; GFX10-NEXT: v_cndmask_b32_e64 v16, v3, v2, s6
-; GFX10-NEXT: v_cndmask_b32_sdwa v21, v4, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s7
-; GFX10-NEXT: v_perm_b32 v4, v1, v0, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v3, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s9
-; GFX10-NEXT: v_perm_b32 v3, v5, v6, 0x5040100
+; GFX10-NEXT: s_not_b32 s0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_sdwa v1, v10, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s11
-; GFX10-NEXT: v_cndmask_b32_e64 v17, v10, v2, s8
-; GFX10-NEXT: v_cndmask_b32_sdwa v5, v9, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s13
-; GFX10-NEXT: v_cndmask_b32_e64 v18, v9, v2, s10
-; GFX10-NEXT: v_cndmask_b32_sdwa v6, v8, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s2
-; GFX10-NEXT: v_cndmask_b32_e64 v19, v8, v2, s12
-; GFX10-NEXT: v_cndmask_b32_e64 v20, v7, v2, s14
-; GFX10-NEXT: v_cndmask_b32_sdwa v9, v7, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v8, v1, v17, 0x5040100
-; GFX10-NEXT: v_perm_b32 v7, v5, v18, 0x5040100
-; GFX10-NEXT: v_perm_b32 v6, v6, v19, 0x5040100
-; GFX10-NEXT: v_perm_b32 v2, v21, v15, 0x5040100
-; GFX10-NEXT: v_perm_b32 v5, v9, v20, 0x5040100
-; GFX10-NEXT: v_perm_b32 v1, v0, v16, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[11:12], v[5:8], off
-; GFX10-NEXT: global_store_dwordx4 v[13:14], v[1:4], off
+; GFX10-NEXT: v_movrels_b32_e32 v1, v3
+; GFX10-NEXT: v_and_or_b32 v2, v1, s0, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_movreld_b32_e32 v3, v2
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX10-NEXT: global_store_dwordx4 v[11:12], v[7:10], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v16i16_v_s:
@@ -5641,61 +4031,24 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
; GFX11-NEXT: global_load_b128 v[7:10], v[0:1], off offset:16
-; GFX11-NEXT: s_cmp_eq_u32 s2, 6
-; GFX11-NEXT: v_dual_mov_b32 v0, 16 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 7
-; GFX11-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 0
-; GFX11-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 4
-; GFX11-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 5
-; GFX11-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 2
-; GFX11-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 3
-; GFX11-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-NEXT: s_cselect_b32 s7, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 1
-; GFX11-NEXT: s_cselect_b32 s8, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 14
-; GFX11-NEXT: s_cselect_b32 s9, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 15
-; GFX11-NEXT: s_cselect_b32 s10, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 12
-; GFX11-NEXT: s_cselect_b32 s11, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 13
-; GFX11-NEXT: s_cselect_b32 s12, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 10
-; GFX11-NEXT: s_cselect_b32 s13, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 11
-; GFX11-NEXT: s_cselect_b32 s14, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 8
-; GFX11-NEXT: s_cselect_b32 s15, -1, 0
-; GFX11-NEXT: s_cmp_eq_u32 s2, 9
-; GFX11-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_cndmask_b16 v6.l, v6.l, v2.l, s0
+; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 1
+; GFX11-NEXT: s_lshr_b32 m0, s2, 1
+; GFX11-NEXT: s_lshl_b32 s0, s0, 4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v11, 16 :: v_dual_lshlrev_b32 v0, s0, v0
+; GFX11-NEXT: s_lshl_b32 s0, 0xffff, s0
+; GFX11-NEXT: v_mov_b32_e32 v12, 0
+; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v10.l, v10.l, v2.l, s9
-; GFX11-NEXT: v_cndmask_b16 v10.h, v10.h, v2.l, s10
-; GFX11-NEXT: v_cndmask_b16 v9.l, v9.l, v2.l, s11
-; GFX11-NEXT: v_cndmask_b16 v9.h, v9.h, v2.l, s12
-; GFX11-NEXT: v_cndmask_b16 v8.l, v8.l, v2.l, s13
-; GFX11-NEXT: v_cndmask_b16 v8.h, v8.h, v2.l, s14
-; GFX11-NEXT: v_cndmask_b16 v7.l, v7.l, v2.l, s15
-; GFX11-NEXT: v_cndmask_b16 v7.h, v7.h, v2.l, s2
-; GFX11-NEXT: v_cndmask_b16 v6.h, v6.h, v2.l, s1
-; GFX11-NEXT: v_cndmask_b16 v5.l, v5.l, v2.l, s3
-; GFX11-NEXT: v_cndmask_b16 v5.h, v5.h, v2.l, s4
-; GFX11-NEXT: v_cndmask_b16 v4.l, v4.l, v2.l, s5
-; GFX11-NEXT: v_cndmask_b16 v4.h, v4.h, v2.l, s6
-; GFX11-NEXT: v_cndmask_b16 v3.l, v3.l, v2.l, s7
-; GFX11-NEXT: v_cndmask_b16 v3.h, v3.h, v2.l, s8
+; GFX11-NEXT: v_movrels_b32_e32 v1, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v2, v1, s0, v0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: v_movreld_b32_e32 v3, v2
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[0:1], v[7:10], off
-; GFX11-NEXT: global_store_b128 v[11:12], v[3:6], off
+; GFX11-NEXT: global_store_b128 v[0:1], v[3:6], off
+; GFX11-NEXT: global_store_b128 v[11:12], v[7:10], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(1) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
@@ -5708,204 +4061,139 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
; GFX9-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v3
-; GFX9-NEXT: s_mov_b32 s8, 0x5040100
-; GFX9-NEXT: v_mov_b32_e32 v0, 16
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX9-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v3, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v1
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v1
; GFX9-NEXT: v_mov_b32_e32 v12, 0
; GFX9-NEXT: v_mov_b32_e32 v13, 0
+; GFX9-NEXT: v_mov_b32_e32 v14, 16
+; GFX9-NEXT: v_mov_b32_e32 v15, 0
; GFX9-NEXT: s_waitcnt vmcnt(1)
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v7, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v7, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v15, v6, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v16, v5, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v17, v4, v2, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v4, v4, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 13, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 14, v3
-; GFX9-NEXT: v_perm_b32 v7, v7, v14, s8
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_e64 v14, v11, v2, s[6:7]
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 12, v3
-; GFX9-NEXT: v_cndmask_b32_sdwa v11, v11, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_perm_b32 v6, v6, v15, s8
-; GFX9-NEXT: v_cndmask_b32_e64 v15, v10, v2, s[6:7]
-; GFX9-NEXT: v_cndmask_b32_sdwa v10, v10, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 10, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 11, v3
-; GFX9-NEXT: v_perm_b32 v11, v11, v14, s8
-; GFX9-NEXT: v_cndmask_b32_e32 v14, v9, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 9, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v8, v2, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_cndmask_b32_sdwa v9, v9, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v8, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_perm_b32 v10, v10, v15, s8
-; GFX9-NEXT: v_perm_b32 v9, v9, v14, s8
-; GFX9-NEXT: v_perm_b32 v8, v2, v3, s8
-; GFX9-NEXT: v_perm_b32 v5, v5, v16, s8
-; GFX9-NEXT: v_perm_b32 v4, v4, v17, s8
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[8:11], off
-; GFX9-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v8, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v10, s[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11]
+; GFX9-NEXT: v_and_or_b32 v16, v3, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, v16, s[12:13]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v16, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v16, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v16, s[2:3]
+; GFX9-NEXT: v_cndmask_b32_e64 v4, v8, v16, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v16, s[6:7]
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v10, v16, s[8:9]
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v16, s[10:11]
+; GFX9-NEXT: global_store_dwordx4 v[12:13], v[0:3], off
+; GFX9-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i16_v_v:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_add_u32_e32 v4, vcc, 16, v0
-; GFX8-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
-; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dwordx4 v[8:11], v[0:1]
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 15, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 14, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 13, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 11, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 9, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v3
+; GFX8-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, 0xffff
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v3, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v1
; GFX8-NEXT: v_mov_b32_e32 v12, 0
; GFX8-NEXT: v_mov_b32_e32 v13, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-NEXT: v_mov_b32_e32 v14, 16
+; GFX8-NEXT: v_mov_b32_e32 v15, 0
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_cndmask_b32_e64 v14, v7, v2, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v7, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v15, v6, v2, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 10, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v16, v5, v2, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v17, v4, v2, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_cndmask_b32_sdwa v6, v6, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v4, v4, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 5, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 6, v3
-; GFX8-NEXT: v_or_b32_sdwa v7, v14, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e64 v14, v11, v2, s[6:7]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 4, v3
-; GFX8-NEXT: v_cndmask_b32_sdwa v11, v11, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v6, v15, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e64 v15, v10, v2, s[6:7]
-; GFX8-NEXT: v_cndmask_b32_sdwa v10, v10, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v3
-; GFX8-NEXT: v_or_b32_sdwa v11, v14, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v14, v9, v2, vcc
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v8, v2, vcc
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_sdwa v9, v9, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_sdwa v2, v8, v2, vcc dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v10, v15, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v9, v14, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v8, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v5, v16, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v4, v17, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v8, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v10, s[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11]
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v3
+; GFX8-NEXT: v_or_b32_e32 v16, v0, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v4, v16, s[12:13]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v16, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v16, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v16, s[2:3]
+; GFX8-NEXT: v_cndmask_b32_e64 v4, v8, v16, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e64 v5, v9, v16, s[6:7]
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v10, v16, s[8:9]
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v11, v16, s[10:11]
+; GFX8-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
+; GFX8-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v16i16_v_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
-; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[0:3], 0 addr64 offset:16
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v3
-; GFX7-NEXT: s_mov_b64 s[0:1], 16
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s18, 0
+; GFX7-NEXT: s_mov_b32 s19, 0xf000
+; GFX7-NEXT: s_mov_b64 s[16:17], 0
+; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[16:19], 0 addr64
+; GFX7-NEXT: buffer_load_dwordx4 v[8:11], v[0:1], s[16:19], 0 addr64 offset:16
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 1, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 4, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[6:7], 5, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[8:9], 6, v0
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 4, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[12:13], 0, v0
+; GFX7-NEXT: s_mov_b64 s[16:17], 0
+; GFX7-NEXT: s_mov_b32 s18, -1
; GFX7-NEXT: s_waitcnt vmcnt(1)
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v6, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v12, v5, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v13, v4, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v3
+; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v14, v11, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v10, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v16, v9, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v17, v8, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v11
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_cndmask_b32_e32 v11, v11, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; GFX7-NEXT: v_and_b32_e32 v13, 0xffff, v13
-; GFX7-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; GFX7-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; GFX7-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX7-NEXT: v_and_b32_e32 v17, 0xffff, v17
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v11
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v10
-; GFX7-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v11, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v0, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v7, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v12, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v13, v4
-; GFX7-NEXT: v_or_b32_e32 v7, v14, v8
-; GFX7-NEXT: v_or_b32_e32 v6, v15, v10
-; GFX7-NEXT: v_or_b32_e32 v5, v16, v9
-; GFX7-NEXT: v_or_b32_e32 v4, v17, v11
-; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v8, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v10, s[8:9]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[10:11]
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v3
+; GFX7-NEXT: v_or_b32_e32 v12, v1, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v4, v12, s[12:13]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v12, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v12, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v12, s[2:3]
+; GFX7-NEXT: v_cndmask_b32_e64 v4, v8, v12, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e64 v5, v9, v12, s[6:7]
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v10, v12, s[8:9]
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v11, v12, s[10:11]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GFX7-NEXT: s_mov_b64 s[16:17], 16
+; GFX7-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i16_v_v:
@@ -5913,61 +4201,44 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
; GFX10-NEXT: global_load_dwordx4 v[8:11], v[0:1], off offset:16
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s7, 6, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 5, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 1, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 15, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 13, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 11, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 9, v3
-; GFX10-NEXT: v_mov_b32_e32 v0, 16
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_mov_b32_e32 v12, 0
+; GFX10-NEXT: v_lshrrev_b32_e32 v0, 1, v3
+; GFX10-NEXT: v_and_b32_e32 v3, 1, v3
; GFX10-NEXT: v_mov_b32_e32 v13, 0
+; GFX10-NEXT: v_mov_b32_e32 v14, 16
+; GFX10-NEXT: v_mov_b32_e32 v15, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 4, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 5, v0
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 6, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 0, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v12, v3, 0xffff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-NEXT: v_not_b32_e32 v3, v12
+; GFX10-NEXT: v_mov_b32_e32 v12, 0
; GFX10-NEXT: s_waitcnt vmcnt(1)
-; GFX10-NEXT: v_cndmask_b32_e64 v14, v7, v2, s7
-; GFX10-NEXT: v_cndmask_b32_sdwa v7, v7, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v15, v6, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v16, v5, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v17, v4, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v3
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v5, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v6, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v7, s1
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e32 v18, v11, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v19, v10, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v20, v9, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v21, v8, v2, vcc_lo
-; GFX10-NEXT: s_mov_b32 vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_sdwa v3, v6, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s1
-; GFX10-NEXT: v_cndmask_b32_sdwa v6, v5, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s2
-; GFX10-NEXT: v_perm_b32 v5, v7, v14, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_sdwa v14, v4, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s3
-; GFX10-NEXT: v_perm_b32 v4, v3, v15, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_sdwa v7, v11, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_perm_b32 v3, v6, v16, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_sdwa v10, v10, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s5
-; GFX10-NEXT: v_cndmask_b32_sdwa v11, v9, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s6
-; GFX10-NEXT: v_perm_b32 v9, v7, v18, 0x5040100
-; GFX10-NEXT: v_cndmask_b32_sdwa v2, v8, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_perm_b32 v8, v10, v19, 0x5040100
-; GFX10-NEXT: v_perm_b32 v7, v11, v20, 0x5040100
-; GFX10-NEXT: v_perm_b32 v6, v2, v21, 0x5040100
-; GFX10-NEXT: v_perm_b32 v2, v14, v17, 0x5040100
-; GFX10-NEXT: global_store_dwordx4 v[0:1], v[6:9], off
-; GFX10-NEXT: global_store_dwordx4 v[12:13], v[2:5], off
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v8, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v10, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v11, s5
+; GFX10-NEXT: v_and_or_b32 v16, v1, v3, v2
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v4, v16, s6
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v16, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v16, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v16, s1
+; GFX10-NEXT: v_cndmask_b32_e64 v4, v8, v16, s2
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v9, v16, s3
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v10, v16, s4
+; GFX10-NEXT: v_cndmask_b32_e64 v7, v11, v16, s5
+; GFX10-NEXT: global_store_dwordx4 v[12:13], v[0:3], off
+; GFX10-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v16i16_v_v:
@@ -5975,46 +4246,49 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
; GFX11-NEXT: global_load_b128 v[8:11], v[0:1], off offset:16
-; GFX11-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-NEXT: v_cmp_eq_u32_e64 s7, 14, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s8, 15, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s9, 12, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s10, 13, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s11, 10, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s12, 11, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s13, 8, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s14, 9, v3
-; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 7, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 4, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 5, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 2, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 3, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 0, v3
-; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 1, v3
-; GFX11-NEXT: v_dual_mov_b32 v12, 16 :: v_dual_mov_b32 v13, 0
-; GFX11-NEXT: v_dual_mov_b32 v14, 0 :: v_dual_mov_b32 v15, 0
+; GFX11-NEXT: v_lshrrev_b32_e32 v0, 1, v3
+; GFX11-NEXT: v_and_b32_e32 v3, 1, v3
+; GFX11-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v14, 16
+; GFX11-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 4, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s3, 5, v0
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 4, v3
+; GFX11-NEXT: v_cmp_eq_u32_e64 s4, 6, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s5, 7, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s6, 0, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_lshlrev_b32 v2, v3, v2
+; GFX11-NEXT: v_lshlrev_b32_e64 v12, v3, 0xffff
+; GFX11-NEXT: v_not_b32_e32 v3, v12
; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_cndmask_b16 v3.l, v7.l, v0.h, vcc_lo
+; GFX11-NEXT: v_dual_mov_b32 v12, 0 :: v_dual_cndmask_b32 v1, v4, v5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v6, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v7, s1
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_cndmask_b16 v11.l, v11.l, v0.h, s7
-; GFX11-NEXT: v_cndmask_b16 v11.h, v11.h, v0.h, s8
-; GFX11-NEXT: v_cndmask_b16 v10.l, v10.l, v0.h, s9
-; GFX11-NEXT: v_cndmask_b16 v10.h, v10.h, v0.h, s10
-; GFX11-NEXT: v_cndmask_b16 v9.l, v9.l, v0.h, s11
-; GFX11-NEXT: v_cndmask_b16 v9.h, v9.h, v0.h, s12
-; GFX11-NEXT: v_cndmask_b16 v8.l, v8.l, v0.h, s13
-; GFX11-NEXT: v_cndmask_b16 v8.h, v8.h, v0.h, s14
-; GFX11-NEXT: v_cndmask_b16 v3.h, v7.h, v0.h, s0
-; GFX11-NEXT: v_cndmask_b16 v2.l, v6.l, v0.h, s1
-; GFX11-NEXT: v_cndmask_b16 v2.h, v6.h, v0.h, s2
-; GFX11-NEXT: v_cndmask_b16 v1.l, v5.l, v0.h, s3
-; GFX11-NEXT: v_cndmask_b16 v1.h, v5.h, v0.h, s4
-; GFX11-NEXT: v_cndmask_b16 v0.l, v4.l, v0.h, s5
-; GFX11-NEXT: v_cndmask_b16 v0.h, v4.h, v0.h, s6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v8, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v9, s3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v10, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v1, v1, v11, s5
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v16, v1, v3, v2
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, v16, s6
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v16, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v16, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v16, s1
+; GFX11-NEXT: v_cndmask_b32_e64 v4, v8, v16, s2
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v9, v16, s3
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v10, v16, s4
+; GFX11-NEXT: v_cndmask_b32_e64 v7, v11, v16, s5
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b128 v[12:13], v[8:11], off
-; GFX11-NEXT: global_store_b128 v[14:15], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[12:13], v[0:3], off
+; GFX11-NEXT: global_store_b128 v[14:15], v[4:7], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i16>, ptr addrspace(1) %ptr
%insert = insertelement <16 x i16> %vec, i16 %val, i32 %idx
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
index 8c4fde1a8d53e..620cba152f5a2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck --check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck --check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-mesa-mesa3d < %s | FileCheck --check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck --check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck --check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01-mesa-mesa3d < %s | FileCheck --check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck --check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck --check-prefixes=GFX11,GFX11-FAKE16 %s
define amdgpu_ps void @insertelement_s_v2i8_s_s(ptr addrspace(4) inreg %ptr, i8 inreg %val, i32 inreg %idx) {
; GFX9-LABEL: insertelement_s_v2i8_s_s:
@@ -1065,38 +1065,33 @@ define amdgpu_ps void @insertelement_v_v4i8_s_s(ptr addrspace(1) %ptr, i8 inreg
; GFX9-LABEL: insertelement_v_v4i8_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v2, v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s2, 8
+; GFX9-NEXT: s_and_b32 s0, s3, 3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 3
; GFX9-NEXT: s_and_b32 s1, s2, 0xff
-; GFX9-NEXT: s_or_b32 s0, s1, s0
-; GFX9-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX9-NEXT: s_lshl_b32 s0, s0, 16
-; GFX9-NEXT: s_or_b32 s0, s1, s0
-; GFX9-NEXT: s_lshl_b32 s1, s3, 3
-; GFX9-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX9-NEXT: s_lshl_b32 s1, s1, s0
+; GFX9-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX9-NEXT: s_not_b32 s0, s0
; GFX9-NEXT: v_mov_b32_e32 v3, s1
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, s0, v2
+; GFX9-NEXT: v_and_or_b32 v2, v2, s0, v3
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i8_s_s:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v2, v[0:1]
-; GFX8-NEXT: s_lshl_b32 s0, s2, 8
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_and_b32 s0, s3, 3
; GFX8-NEXT: s_and_b32 s1, s2, 0xff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_lshl_b32 s1, s0, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s1, s3, 3
-; GFX8-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, s0, v2
+; GFX8-NEXT: v_bfi_b32 v2, s0, 0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_or_b32_e32 v2, s1, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -1104,58 +1099,49 @@ define amdgpu_ps void @insertelement_v_v4i8_s_s(ptr addrspace(1) %ptr, i8 inreg
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 8
+; GFX7-NEXT: s_and_b32 s0, s3, 3
; GFX7-NEXT: s_and_b32 s1, s2, 0xff
-; GFX7-NEXT: s_lshl_b32 s2, s3, 3
-; GFX7-NEXT: s_or_b32 s0, s1, s0
-; GFX7-NEXT: s_lshl_b32 s1, 0xff, s2
-; GFX7-NEXT: s_lshl_b32 s2, s0, 16
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: s_or_b32 s0, s0, s2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 3
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, s0, v0
+; GFX7-NEXT: v_bfi_b32 v0, s0, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, s1, v0
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i8_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v2, v[0:1], off
-; GFX10-NEXT: s_lshl_b32 s0, s2, 8
+; GFX10-NEXT: s_and_b32 s0, s3, 3
; GFX10-NEXT: s_and_b32 s1, s2, 0xff
-; GFX10-NEXT: s_lshl_b32 s2, s3, 3
-; GFX10-NEXT: s_or_b32 s0, s1, s0
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX10-NEXT: s_lshl_b32 s0, s0, 16
+; GFX10-NEXT: s_lshl_b32 s2, 0xff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_or_b32 s0, s1, s0
-; GFX10-NEXT: s_lshl_b32 s1, 0xff, s2
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, s1, s0, v2
+; GFX10-NEXT: v_and_or_b32 v2, v2, s1, s0
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v4i8_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v2, v[0:1], off
-; GFX11-NEXT: s_lshl_b32 s0, s2, 8
+; GFX11-NEXT: s_and_b32 s0, s3, 3
; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: s_lshl_b32 s2, s3, 3
-; GFX11-NEXT: s_or_b32 s0, s1, s0
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_lshl_b32 s1, 0xff, s2
+; GFX11-NEXT: s_lshl_b32 s2, 0xff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, s1, s0, v2
+; GFX11-NEXT: v_and_or_b32 v2, v2, s1, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(1 ) %ptr
@@ -1168,75 +1154,79 @@ define amdgpu_ps void @insertelement_s_v4i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v4i8_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: s_mov_b32 s1, 0x4040404
-; GFX9-NEXT: v_perm_b32 v2, v0, v0, s1
-; GFX9-NEXT: s_lshl_b32 s1, s4, 3
-; GFX9-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: s_and_b32 s1, s4, 3
+; GFX9-NEXT: s_lshl_b32 s1, s1, 3
+; GFX9-NEXT: s_lshl_b32 s2, 0xff, s1
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_andn2_b32 s0, s0, s2
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, s1, v3
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v4i8_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: s_mov_b32 s1, 0x4040404
-; GFX8-NEXT: v_perm_b32 v2, v0, v0, s1
-; GFX8-NEXT: s_lshl_b32 s1, s4, 3
+; GFX8-NEXT: s_and_b32 s1, s4, 3
+; GFX8-NEXT: s_lshl_b32 s1, s1, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_andn2_b32 s0, s0, s1
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX8-NEXT: v_or_b32_e32 v2, s0, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v4i8_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; GFX7-NEXT: s_and_b32 s1, s4, 3
+; GFX7-NEXT: s_lshl_b32 s1, s1, 3
; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_lshl_b32 s1, s4, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s1, v0
; GFX7-NEXT: s_lshl_b32 s1, 0xff, s1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, v0, s0
+; GFX7-NEXT: s_andn2_b32 s0, s0, s1
+; GFX7-NEXT: v_or_b32_e32 v0, s0, v0
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v4i8_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX10-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX10-NEXT: s_lshl_b32 s1, s4, 3
+; GFX10-NEXT: s_and_b32 s1, s4, 3
+; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX10-NEXT: s_lshl_b32 s1, s1, 3
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX10-NEXT: s_lshl_b32 s2, 0xff, s1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, s1, v2, s0
+; GFX10-NEXT: s_andn2_b32 s0, s0, s2
+; GFX10-NEXT: v_lshl_or_b32 v2, v2, s1, s0
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i8_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX11-NEXT: s_lshl_b32 s1, s4, 3
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX11-NEXT: s_and_b32 s1, s4, 3
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xff, v0
+; GFX11-NEXT: s_lshl_b32 s1, s1, 3
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_lshl_b32 s2, 0xff, s1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, s1, v2, s0
+; GFX11-NEXT: s_and_not1_b32 s0, s0, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshl_or_b32 v2, v2, s1, s0
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(4) %ptr
@@ -1249,96 +1239,85 @@ define amdgpu_ps void @insertelement_s_v4i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v4i8_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: s_lshl_b32 s1, s4, 8
-; GFX9-NEXT: s_and_b32 s3, s4, 0xff
-; GFX9-NEXT: s_or_b32 s1, s3, s1
-; GFX9-NEXT: s_movk_i32 s2, 0xff
-; GFX9-NEXT: s_and_b32 s3, s1, 0xffff
-; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT: s_or_b32 s1, s3, s1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-NEXT: v_lshlrev_b32_e64 v3, v0, s2
+; GFX9-NEXT: s_and_b32 s1, s4, 0xff
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, v0, s1
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_not_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_bfi_b32 v2, v3, s1, v2
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_and_or_b32 v2, s0, v3, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v4i8_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: s_lshl_b32 s1, s4, 8
-; GFX8-NEXT: s_and_b32 s3, s4, 0xff
-; GFX8-NEXT: s_or_b32 s1, s3, s1
-; GFX8-NEXT: s_movk_i32 s2, 0xff
-; GFX8-NEXT: s_lshl_b32 s3, s1, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX8-NEXT: s_or_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s1, s4, 0xff
+; GFX8-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v0, s2
+; GFX8-NEXT: v_bfi_b32 v3, v0, 0, s0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_bfi_b32 v2, v3, s1, v2
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v4i8_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s1, s4, 8
-; GFX7-NEXT: s_and_b32 s2, s4, 0xff
-; GFX7-NEXT: s_or_b32 s1, s2, s1
-; GFX7-NEXT: s_lshl_b32 s2, s1, 16
-; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX7-NEXT: v_and_b32_e32 v0, 3, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX7-NEXT: s_or_b32 s1, s1, s2
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: s_and_b32 s1, s4, 0xff
+; GFX7-NEXT: v_lshl_b32_e32 v1, s1, v0
; GFX7-NEXT: v_lshl_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_bfi_b32 v0, v0, s1, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, s0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v4i8_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX10-NEXT: s_and_b32 s1, s4, 0xff
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX10-NEXT: s_lshl_b32 s1, s4, 8
-; GFX10-NEXT: s_and_b32 s2, s4, 0xff
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_or_b32 s1, s2, s1
-; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, 0xff
-; GFX10-NEXT: s_and_b32 s2, s1, 0xffff
-; GFX10-NEXT: s_lshl_b32 s1, s1, 16
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v0, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, s1
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_or_b32 s1, s2, s1
+; GFX10-NEXT: v_not_b32_e32 v3, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v2, s1, s0
+; GFX10-NEXT: v_and_or_b32 v2, s0, v3, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i8_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 3, v0
-; GFX11-NEXT: s_lshl_b32 s1, s4, 8
-; GFX11-NEXT: s_and_b32 s2, s4, 0xff
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s1, s2, s1
-; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xff
-; GFX11-NEXT: s_and_b32 s2, s1, 0xffff
-; GFX11-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX11-NEXT: s_and_b32 s1, s4, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v0, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, s1
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_or_b32 s1, s2, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v3, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v2, s1, s0
+; GFX11-NEXT: v_and_or_b32 v2, s0, v3, v2
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(4) %ptr
@@ -1351,76 +1330,82 @@ define amdgpu_ps void @insertelement_s_v4i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v4i8_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX9-NEXT: s_mov_b32 s1, 0x4040404
-; GFX9-NEXT: v_perm_b32 v2, v0, v0, s1
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v1
-; GFX9-NEXT: s_movk_i32 s1, 0xff
-; GFX9-NEXT: v_lshlrev_b32_e64 v3, v0, s1
+; GFX9-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v1, v2
+; GFX9-NEXT: v_not_b32_e32 v2, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX9-NEXT: v_and_or_b32 v2, s0, v2, v3
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v4i8_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX8-NEXT: s_mov_b32 s1, 0x4040404
-; GFX8-NEXT: v_perm_b32 v2, v0, v0, s1
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v1
-; GFX8-NEXT: s_movk_i32 s1, 0xff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v0, s1
+; GFX8-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v1, v2
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_bfi_b32 v2, v0, 0, s0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v4i8_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, v0, s0
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, s0
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v4i8_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dword s0, s[2:3], 0x0
+; GFX10-NEXT: v_and_b32_e32 v1, 3, v1
; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX10-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v1, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v1, 0xff
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_not_b32_e32 v2, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX10-NEXT: v_and_or_b32 v2, s0, v2, v3
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v4i8_v_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b32 s0, s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e64 v3, v1, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v1, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_lshlrev_b32 v3, v1, v0
; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: v_not_b32_e32 v2, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v3, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v2, s0, v2, v3
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(4) %ptr
@@ -1433,38 +1418,34 @@ define amdgpu_ps void @insertelement_v_v4i8_s_v(ptr addrspace(1) %ptr, i8 inreg
; GFX9-LABEL: insertelement_v_v4i8_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v3, v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s2, 8
-; GFX9-NEXT: s_and_b32 s2, s2, 0xff
-; GFX9-NEXT: s_or_b32 s0, s2, s0
-; GFX9-NEXT: s_movk_i32 s1, 0xff
-; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
-; GFX9-NEXT: s_lshl_b32 s0, s0, 16
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX9-NEXT: s_or_b32 s0, s2, s0
-; GFX9-NEXT: v_lshlrev_b32_e64 v2, v0, s1
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT: s_and_b32 s0, s2, 0xff
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_lshlrev_b32_e64 v2, v0, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_not_b32_e32 v4, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX9-NEXT: v_and_or_b32 v2, v3, v4, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i8_s_v:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
-; GFX8-NEXT: s_lshl_b32 s0, s2, 8
-; GFX8-NEXT: s_and_b32 s2, s2, 0xff
-; GFX8-NEXT: s_or_b32 s0, s2, s0
-; GFX8-NEXT: s_movk_i32 s1, 0xff
-; GFX8-NEXT: s_lshl_b32 s2, s0, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX8-NEXT: s_or_b32 s0, s0, s2
-; GFX8-NEXT: v_lshlrev_b32_e64 v2, v0, s1
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX8-NEXT: s_and_b32 s0, s2, 0xff
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v2, v1
+; GFX8-NEXT: v_lshlrev_b32_e64 v3, v2, s0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfi_b32 v2, v1, 0, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -1472,57 +1453,52 @@ define amdgpu_ps void @insertelement_v_v4i8_s_v(ptr addrspace(1) %ptr, i8 inreg
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 8
-; GFX7-NEXT: s_and_b32 s1, s2, 0xff
-; GFX7-NEXT: s_or_b32 s0, s1, s0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v2
-; GFX7-NEXT: s_lshl_b32 s1, s0, 16
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xff
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX7-NEXT: v_lshl_b32_e32 v2, s0, v1
; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v1, s0, v0
+; GFX7-NEXT: v_bfi_b32 v0, v1, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i8_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v2
-; GFX10-NEXT: s_lshl_b32 s0, s2, 8
-; GFX10-NEXT: s_and_b32 s1, s2, 0xff
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_or_b32 s0, s1, s0
-; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, 0xff
-; GFX10-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX10-NEXT: s_lshl_b32 s0, s0, 16
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v2
+; GFX10-NEXT: s_and_b32 s0, s2, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v0, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v0, s0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_or_b32 s0, s1, s0
+; GFX10-NEXT: v_not_b32_e32 v4, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX10-NEXT: v_and_or_b32 v2, v3, v4, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v4i8_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 3, v2
-; GFX11-NEXT: s_lshl_b32 s0, s2, 8
-; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xff
-; GFX11-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v1, v0, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, s0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_or_b32 s0, s1, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v4, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v2, v2, s0, v3
+; GFX11-NEXT: v_and_or_b32 v2, v3, v4, v2
; GFX11-NEXT: global_store_b32 v[0:1], v2, off
; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(1) %ptr
@@ -1535,28 +1511,31 @@ define amdgpu_ps void @insertelement_v_v4i8_v_s(ptr addrspace(1) %ptr, i8 %val,
; GFX9-LABEL: insertelement_v_v4i8_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v3, v[0:1], off
-; GFX9-NEXT: s_mov_b32 s0, 0x4040404
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX9-NEXT: s_lshl_b32 s0, s2, 3
+; GFX9-NEXT: s_and_b32 s0, s2, 3
+; GFX9-NEXT: s_lshl_b32 s0, s0, 3
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX9-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX9-NEXT: s_not_b32 s0, s0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, s0, v2, v3
+; GFX9-NEXT: v_and_or_b32 v2, v3, s0, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i8_v_s:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v3, v[0:1]
-; GFX8-NEXT: s_mov_b32 s0, 0x4040404
-; GFX8-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX8-NEXT: s_lshl_b32 s0, s2, 3
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: s_and_b32 s0, s2, 3
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
; GFX8-NEXT: s_lshl_b32 s0, 0xff, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, s0, v2, v3
+; GFX8-NEXT: v_bfi_b32 v3, s0, 0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -1564,63 +1543,52 @@ define amdgpu_ps void @insertelement_v_v4i8_v_s(ptr addrspace(1) %ptr, i8 %val,
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_lshl_b32 s0, s2, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: s_and_b32 s0, s2, 3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, s0, v1
; GFX7-NEXT: s_lshl_b32 s0, 0xff, s0
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, s0, v1, v0
+; GFX7-NEXT: v_bfi_b32 v0, s0, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i8_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v3, v[0:1], off
-; GFX10-NEXT: v_perm_b32 v2, v2, v2, 0x4040404
-; GFX10-NEXT: s_lshl_b32 s0, s2, 3
+; GFX10-NEXT: s_and_b32 s0, s2, 3
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX10-NEXT: s_not_b32 s0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, s0, v2, v3
+; GFX10-NEXT: v_and_or_b32 v2, v3, s0, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v4i8_v_s:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s2, 3
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, 0xff, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, s0, v2, v3
-; GFX11-TRUE16-NEXT: global_store_b32 v[0:1], v2, off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v4i8_v_s:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b32 v3, v[0:1], off
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v2, 0x4040404
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s2, 3
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, 0xff, s0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v2, s0, v2, v3
-; GFX11-FAKE16-NEXT: global_store_b32 v[0:1], v2, off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v4i8_v_s:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b32 v3, v[0:1], off
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v0, 0xff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, s0, v0
+; GFX11-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-NEXT: s_not_b32 s0, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_and_or_b32 v2, v3, s0, v2
+; GFX11-NEXT: global_store_b32 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <4 x i8> %vec, i8 %val, i32 %idx
store <4 x i8> %insert, ptr addrspace(1) null
@@ -1631,30 +1599,32 @@ define amdgpu_ps void @insertelement_v_v4i8_v_v(ptr addrspace(1) %ptr, i8 %val,
; GFX9-LABEL: insertelement_v_v4i8_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dword v4, v[0:1], off
-; GFX9-NEXT: s_mov_b32 s0, 0x4040404
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v3
-; GFX9-NEXT: s_movk_i32 s0, 0xff
-; GFX9-NEXT: v_lshlrev_b32_e64 v3, v0, s0
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v1
+; GFX9-NEXT: v_not_b32_e32 v3, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v3, v2, v4
+; GFX9-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX9-NEXT: global_store_dword v[0:1], v2, off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v4i8_v_v:
; GFX8: ; %bb.0:
-; GFX8-NEXT: flat_load_dword v4, v[0:1]
-; GFX8-NEXT: s_mov_b32 s0, 0x4040404
-; GFX8-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v3
-; GFX8-NEXT: s_movk_i32 s0, 0xff
-; GFX8-NEXT: v_lshlrev_b32_e64 v3, v0, s0
+; GFX8-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX8-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_bfi_b32 v3, v1, 0, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v3, v2, v4
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -1662,65 +1632,53 @@ define amdgpu_ps void @insertelement_v_v4i8_v_v(ptr addrspace(1) %ptr, i8 %val,
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s2, 0
; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
+; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v2
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: v_lshl_b32_e32 v2, 0xff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
; GFX7-NEXT: s_mov_b64 s[0:1], 0
; GFX7-NEXT: s_mov_b32 s2, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v2, v1, v0
+; GFX7-NEXT: v_bfi_b32 v0, v1, 0, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v4i8_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dword v4, v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v3
-; GFX10-NEXT: v_perm_b32 v2, v2, v2, 0x4040404
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_lshlrev_b32_e64 v3, v0, 0xff
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_lshlrev_b32_e64 v1, v0, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_not_b32_e32 v3, v1
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v3, v2, v4
+; GFX10-NEXT: v_and_or_b32 v2, v4, v3, v2
; GFX10-NEXT: global_store_dword v[0:1], v2, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v4i8_v_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v3
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e64 v3, v1, 0xff
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v2, v3, v2, v4
-; GFX11-TRUE16-NEXT: global_store_b32 v[0:1], v2, off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v4i8_v_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b32 v4, v[0:1], off
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 3, v3
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v2, 0x4040404
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e64 v3, v0, 0xff
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v2, v3, v2, v4
-; GFX11-FAKE16-NEXT: global_store_b32 v[0:1], v2, off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v4i8_v_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b32 v4, v[0:1], off
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v3
+; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v0, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_lshlrev_b32 v3, v0, v1
+; GFX11-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NEXT: v_not_b32_e32 v2, v2
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v2, v4, v2, v3
+; GFX11-NEXT: global_store_b32 v[0:1], v2, off
+; GFX11-NEXT: s_endpgm
%vec = load <4 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <4 x i8> %vec, i8 %val, i32 %idx
store <4 x i8> %insert, ptr addrspace(1) null
@@ -1731,19 +1689,17 @@ define amdgpu_ps void @insertelement_s_v8i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v8i8_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX9-NEXT: s_lshl_b32 s2, s4, 8
-; GFX9-NEXT: s_and_b32 s3, s4, 0xff
-; GFX9-NEXT: s_or_b32 s2, s3, s2
-; GFX9-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX9-NEXT: s_lshl_b32 s2, s2, 16
-; GFX9-NEXT: s_or_b32 s2, s3, s2
-; GFX9-NEXT: s_mov_b32 s3, s2
-; GFX9-NEXT: s_lshl_b32 s4, s5, 3
+; GFX9-NEXT: s_and_b32 s3, s5, 3
+; GFX9-NEXT: s_lshr_b32 m0, s5, 2
+; GFX9-NEXT: s_lshl_b32 s3, s3, 3
+; GFX9-NEXT: s_and_b32 s4, s4, 0xff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GFX9-NEXT: s_lshl_b64 s[4:5], 0xff, s4
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
-; GFX9-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT: s_movrels_b32 s2, s0
+; GFX9-NEXT: s_lshl_b32 s4, s4, s3
+; GFX9-NEXT: s_lshl_b32 s3, 0xff, s3
+; GFX9-NEXT: s_andn2_b32 s2, s2, s3
+; GFX9-NEXT: s_or_b32 s2, s2, s4
+; GFX9-NEXT: s_movreld_b32 s0, s2
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_mov_b32_e32 v3, s1
@@ -1754,19 +1710,17 @@ define amdgpu_ps void @insertelement_s_v8i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX8-LABEL: insertelement_s_v8i8_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: s_lshl_b32 s2, s4, 8
-; GFX8-NEXT: s_and_b32 s3, s4, 0xff
-; GFX8-NEXT: s_or_b32 s2, s3, s2
-; GFX8-NEXT: s_lshl_b32 s3, s2, 16
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s2, s2, s3
-; GFX8-NEXT: s_mov_b32 s3, s2
-; GFX8-NEXT: s_lshl_b32 s4, s5, 3
+; GFX8-NEXT: s_and_b32 s3, s5, 3
+; GFX8-NEXT: s_lshr_b32 m0, s5, 2
+; GFX8-NEXT: s_lshl_b32 s3, s3, 3
+; GFX8-NEXT: s_and_b32 s4, s4, 0xff
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GFX8-NEXT: s_lshl_b64 s[4:5], 0xff, s4
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
+; GFX8-NEXT: s_movrels_b32 s2, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, s3
+; GFX8-NEXT: s_lshl_b32 s3, 0xff, s3
+; GFX8-NEXT: s_andn2_b32 s2, s2, s3
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_movreld_b32 s0, s2
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
@@ -1776,46 +1730,42 @@ define amdgpu_ps void @insertelement_s_v8i8_s_s(ptr addrspace(4) inreg %ptr, i8
;
; GFX7-LABEL: insertelement_s_v8i8_s_s:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s2, s4, 8
-; GFX7-NEXT: s_and_b32 s3, s4, 0xff
-; GFX7-NEXT: s_or_b32 s2, s3, s2
-; GFX7-NEXT: s_lshl_b32 s3, s2, 16
-; GFX7-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX7-NEXT: s_or_b32 s2, s2, s3
-; GFX7-NEXT: s_mov_b32 s3, s2
-; GFX7-NEXT: s_lshl_b32 s4, s5, 3
+; GFX7-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; GFX7-NEXT: s_and_b32 s1, s5, 3
+; GFX7-NEXT: s_lshr_b32 m0, s5, 2
+; GFX7-NEXT: s_lshl_b32 s1, s1, 3
+; GFX7-NEXT: s_and_b32 s4, s4, 0xff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GFX7-NEXT: s_lshl_b64 s[4:5], 0xff, s4
-; GFX7-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
-; GFX7-NEXT: s_xor_b64 s[4:5], s[2:3], s[0:1]
+; GFX7-NEXT: s_movrels_b32 s0, s2
+; GFX7-NEXT: s_lshl_b32 s4, s4, s1
+; GFX7-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX7-NEXT: s_andn2_b32 s0, s0, s1
+; GFX7-NEXT: s_or_b32 s0, s0, s4
+; GFX7-NEXT: s_movreld_b32 s2, s0
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i8_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: s_lshl_b32 s2, s4, 8
+; GFX10-NEXT: s_and_b32 s2, s5, 3
+; GFX10-NEXT: s_lshr_b32 m0, s5, 2
+; GFX10-NEXT: s_lshl_b32 s2, s2, 3
; GFX10-NEXT: s_and_b32 s3, s4, 0xff
-; GFX10-NEXT: s_lshl_b32 s4, s5, 3
-; GFX10-NEXT: s_or_b32 s2, s3, s2
-; GFX10-NEXT: s_lshl_b64 s[4:5], 0xff, s4
-; GFX10-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_lshl_b32 s5, 0xff, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, s2
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_or_b32 s2, s3, s2
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: s_mov_b32 s3, s2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GFX10-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
-; GFX10-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT: s_movrels_b32 s4, s0
+; GFX10-NEXT: s_andn2_b32 s3, s4, s5
+; GFX10-NEXT: s_or_b32 s2, s3, s2
+; GFX10-NEXT: s_movreld_b32 s0, s2
; GFX10-NEXT: v_mov_b32_e32 v3, s1
; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
@@ -1824,22 +1774,20 @@ define amdgpu_ps void @insertelement_s_v8i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX11-LABEL: insertelement_s_v8i8_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: s_lshl_b32 s2, s4, 8
+; GFX11-NEXT: s_and_b32 s2, s5, 3
+; GFX11-NEXT: s_lshr_b32 m0, s5, 2
+; GFX11-NEXT: s_lshl_b32 s2, s2, 3
; GFX11-NEXT: s_and_b32 s3, s4, 0xff
-; GFX11-NEXT: s_lshl_b32 s4, s5, 3
-; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: s_lshl_b64 s[4:5], 0xff, s4
-; GFX11-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX11-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-NEXT: s_lshl_b32 s5, 0xff, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, s2
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mov_b32 s3, s2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_xor_b64 s[2:3], s[2:3], s[0:1]
-; GFX11-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
+; GFX11-NEXT: s_movrels_b32 s4, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-NEXT: s_and_not1_b32 s3, s4, s5
+; GFX11-NEXT: s_or_b32 s2, s3, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_movreld_b32 s0, s2
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX11-NEXT: s_endpgm
@@ -1853,42 +1801,43 @@ define amdgpu_ps void @insertelement_v_v8i8_s_s(ptr addrspace(1) %ptr, i8 inreg
; GFX9-LABEL: insertelement_v_v8i8_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s0, s2, 8
-; GFX9-NEXT: s_and_b32 s1, s2, 0xff
-; GFX9-NEXT: s_lshl_b32 s2, s3, 3
-; GFX9-NEXT: s_or_b32 s3, s1, s0
-; GFX9-NEXT: s_lshl_b64 s[0:1], 0xff, s2
-; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
-; GFX9-NEXT: s_lshl_b32 s3, s3, 16
-; GFX9-NEXT: s_or_b32 s2, s2, s3
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
+; GFX9-NEXT: s_and_b32 s1, s3, 3
+; GFX9-NEXT: s_and_b32 s2, s2, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 3
+; GFX9-NEXT: s_lshr_b32 s0, s3, 2
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
+; GFX9-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v4, v3, s1, v2
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX9-NEXT: v_bfi_b32 v0, s0, v5, v0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i8_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_lshl_b32 s0, s2, 8
+; GFX8-NEXT: s_and_b32 s0, s3, 3
+; GFX8-NEXT: s_lshr_b32 m0, s3, 2
; GFX8-NEXT: s_and_b32 s1, s2, 0xff
-; GFX8-NEXT: s_lshl_b32 s2, s3, 3
-; GFX8-NEXT: s_or_b32 s3, s1, s0
-; GFX8-NEXT: s_lshl_b64 s[0:1], 0xff, s2
-; GFX8-NEXT: s_lshl_b32 s2, s3, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_or_b32 s2, s3, s2
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX8-NEXT: v_bfi_b32 v0, s0, v5, v0
+; GFX8-NEXT: v_movrels_b32_e32 v2, v0
+; GFX8-NEXT: v_bfi_b32 v2, s0, 0, v2
+; GFX8-NEXT: v_or_b32_e32 v4, s1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_movreld_b32_e32 v0, v4
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -1896,61 +1845,60 @@ define amdgpu_ps void @insertelement_v_v8i8_s_s(ptr addrspace(1) %ptr, i8 inreg
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s0, s2, 8
+; GFX7-NEXT: s_and_b32 s0, s3, 3
+; GFX7-NEXT: s_lshr_b32 m0, s3, 2
; GFX7-NEXT: s_and_b32 s1, s2, 0xff
-; GFX7-NEXT: s_lshl_b32 s2, s3, 3
-; GFX7-NEXT: s_or_b32 s3, s1, s0
-; GFX7-NEXT: s_lshl_b64 s[0:1], 0xff, s2
-; GFX7-NEXT: s_lshl_b32 s2, s3, 16
-; GFX7-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX7-NEXT: s_or_b32 s2, s3, s2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 3
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX7-NEXT: v_bfi_b32 v0, s0, v3, v0
+; GFX7-NEXT: v_movrels_b32_e32 v2, v0
+; GFX7-NEXT: v_bfi_b32 v2, s0, 0, v2
+; GFX7-NEXT: v_or_b32_e32 v2, s1, v2
+; GFX7-NEXT: v_movreld_b32_e32 v0, v2
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i8_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: s_lshl_b32 s0, s2, 8
+; GFX10-NEXT: s_and_b32 s0, s3, 3
+; GFX10-NEXT: s_lshr_b32 m0, s3, 2
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
; GFX10-NEXT: s_and_b32 s1, s2, 0xff
-; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: s_or_b32 s0, s1, s0
-; GFX10-NEXT: s_lshl_b32 s1, s3, 3
-; GFX10-NEXT: s_and_b32 s2, s0, 0xffff
-; GFX10-NEXT: s_lshl_b32 s3, s0, 16
-; GFX10-NEXT: s_lshl_b64 s[0:1], 0xff, s1
-; GFX10-NEXT: s_or_b32 s2, s2, s3
+; GFX10-NEXT: s_lshl_b32 s2, 0xff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, s1, s2, v1
-; GFX10-NEXT: v_bfi_b32 v0, s0, s2, v0
+; GFX10-NEXT: v_movrels_b32_e32 v2, v0
+; GFX10-NEXT: v_and_or_b32 v4, v2, s1, s0
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v4
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i8_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_lshl_b32 s0, s2, 8
+; GFX11-NEXT: s_and_b32 s0, s3, 3
+; GFX11-NEXT: s_lshr_b32 m0, s3, 2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: s_lshl_b32 s1, s3, 3
-; GFX11-NEXT: s_and_b32 s2, s0, 0xffff
-; GFX11-NEXT: s_lshl_b32 s3, s0, 16
-; GFX11-NEXT: s_lshl_b64 s[0:1], 0xff, s1
-; GFX11-NEXT: s_or_b32 s2, s2, s3
+; GFX11-NEXT: s_lshl_b32 s2, 0xff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, s1, s2, v1
-; GFX11-NEXT: v_bfi_b32 v0, s0, s2, v0
+; GFX11-NEXT: v_movrels_b32_e32 v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v4, v2, s1, s0
+; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v4
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(1 ) %ptr
@@ -1963,84 +1911,108 @@ define amdgpu_ps void @insertelement_s_v8i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v8i8_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX9-NEXT: s_lshl_b32 s2, s4, 3
-; GFX9-NEXT: s_mov_b32 s4, 0x4040404
-; GFX9-NEXT: s_lshl_b64 s[2:3], 0xff, s2
-; GFX9-NEXT: v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT: s_lshr_b32 s2, s4, 2
+; GFX9-NEXT: s_and_b32 s4, s4, 3
+; GFX9-NEXT: s_mov_b32 m0, s2
+; GFX9-NEXT: s_lshl_b32 s4, s4, 3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_movrels_b32 s3, s0
+; GFX9-NEXT: s_lshl_b32 s5, 0xff, s4
+; GFX9-NEXT: s_andn2_b32 s3, s3, s5
+; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_lshl_or_b32 v4, v0, s4, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_bfi_b32 v1, s3, v0, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_bfi_b32 v0, s2, v0, v4
+; GFX9-NEXT: s_set_gpr_idx_on s2, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v8i8_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: s_lshl_b32 s2, s4, 3
-; GFX8-NEXT: s_mov_b32 s4, 0x4040404
-; GFX8-NEXT: s_lshl_b64 s[2:3], 0xff, s2
-; GFX8-NEXT: v_perm_b32 v0, v0, v0, s4
+; GFX8-NEXT: s_and_b32 s3, s4, 3
+; GFX8-NEXT: s_lshr_b32 m0, s4, 2
+; GFX8-NEXT: s_lshl_b32 s3, s3, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_movrels_b32 s2, s0
+; GFX8-NEXT: s_lshl_b32 s3, 0xff, s3
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_andn2_b32 s2, s2, s3
+; GFX8-NEXT: v_or_b32_e32 v4, s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_bfi_b32 v1, s3, v0, v1
; GFX8-NEXT: v_mov_b32_e32 v2, 0
; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_bfi_b32 v0, s2, v0, v4
+; GFX8-NEXT: v_movreld_b32_e32 v0, v4
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v8i8_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v0
+; GFX7-NEXT: s_and_b32 s3, s4, 3
+; GFX7-NEXT: s_lshr_b32 m0, s4, 2
+; GFX7-NEXT: s_lshl_b32 s3, s3, 3
; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_lshl_b32 s2, s4, 3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_lshl_b64 s[2:3], 0xff, s2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_movrels_b32 s2, s0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX7-NEXT: s_lshl_b32 s3, 0xff, s3
+; GFX7-NEXT: s_andn2_b32 s2, s2, s3
+; GFX7-NEXT: v_or_b32_e32 v2, s2, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_bfi_b32 v1, s3, v0, v1
-; GFX7-NEXT: v_bfi_b32 v0, s2, v0, v2
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_movreld_b32_e32 v0, v2
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i8_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX10-NEXT: s_lshl_b32 s2, s4, 3
-; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_lshl_b64 s[2:3], 0xff, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: s_and_b32 s2, s4, 3
+; GFX10-NEXT: s_lshr_b32 m0, s4, 2
+; GFX10-NEXT: s_lshl_b32 s2, s2, 3
+; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v3, s3, v2, s1
-; GFX10-NEXT: v_bfi_b32 v2, s2, v2, s0
-; GFX10-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX10-NEXT: s_movrels_b32 s4, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_andn2_b32 s3, s4, s3
+; GFX10-NEXT: v_lshl_or_b32 v4, v0, s2, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v4
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i8_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_perm_b32 v2, v0, v0, 0x4040404
-; GFX11-NEXT: s_lshl_b32 s2, s4, 3
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
-; GFX11-NEXT: s_lshl_b64 s[2:3], 0xff, s2
+; GFX11-NEXT: s_and_b32 s2, s4, 3
+; GFX11-NEXT: s_lshr_b32 m0, s4, 2
+; GFX11-NEXT: s_lshl_b32 s2, s2, 3
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT: s_lshl_b32 s3, 0xff, s2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v3, s3, v2, s1
-; GFX11-NEXT: v_bfi_b32 v2, s2, v2, s0
-; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
+; GFX11-NEXT: s_movrels_b32 s4, s0
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_and_not1_b32 s3, s4, s3
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_lshl_or_b32 v4, v0, s2, s3
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v4
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i8> %vec, i8 %val, i32 %idx
@@ -2052,105 +2024,124 @@ define amdgpu_ps void @insertelement_s_v8i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v8i8_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX9-NEXT: s_lshl_b32 s2, s4, 8
-; GFX9-NEXT: s_and_b32 s3, s4, 0xff
-; GFX9-NEXT: s_or_b32 s2, s3, s2
-; GFX9-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX9-NEXT: s_lshl_b32 s2, s2, 16
-; GFX9-NEXT: s_or_b32 s4, s3, s2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 2, v0
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX9-NEXT: s_mov_b64 s[2:3], 0xff
-; GFX9-NEXT: v_lshlrev_b64 v[0:1], v0, s[2:3]
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: v_bfi_b32 v1, v1, s4, v2
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: s_and_b32 s0, s4, 0xff
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX9-NEXT: v_lshlrev_b32_e64 v5, v0, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_and_or_b32 v5, v4, v0, v5
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v1, v5, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_bfi_b32 v0, v0, s4, v4
-; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
+; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX9-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v8i8_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX8-NEXT: s_lshl_b32 s2, s4, 8
-; GFX8-NEXT: s_and_b32 s3, s4, 0xff
-; GFX8-NEXT: s_or_b32 s2, s3, s2
-; GFX8-NEXT: s_lshl_b32 s3, s2, 16
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s4, s2, s3
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 2, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX8-NEXT: s_mov_b64 s[2:3], 0xff
-; GFX8-NEXT: v_lshlrev_b64 v[0:1], v0, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: v_bfi_b32 v1, v1, s4, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: s_and_b32 s0, s4, 0xff
+; GFX8-NEXT: v_mov_b32_e32 v6, 0xff
+; GFX8-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX8-NEXT: v_lshlrev_b32_e64 v5, v0, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v6
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v4
+; GFX8-NEXT: v_or_b32_e32 v5, v0, v5
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v1, v5, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_bfi_b32 v0, v0, s4, v4
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v5, vcc
+; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v8i8_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: s_lshl_b32 s2, s4, 8
-; GFX7-NEXT: s_and_b32 s3, s4, 0xff
-; GFX7-NEXT: s_or_b32 s2, s3, s2
-; GFX7-NEXT: s_lshl_b32 s3, s2, 16
-; GFX7-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX7-NEXT: s_or_b32 s4, s2, s3
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 2, v0
+; GFX7-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX7-NEXT: s_mov_b64 s[2:3], 0xff
-; GFX7-NEXT: v_lshl_b64 v[0:1], s[2:3], v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v2, s1
-; GFX7-NEXT: v_bfi_b32 v1, v1, s4, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
-; GFX7-NEXT: v_bfi_b32 v0, v0, s4, v2
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: s_and_b32 s0, s4, 0xff
+; GFX7-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc
+; GFX7-NEXT: v_lshl_b32_e32 v5, s0, v0
+; GFX7-NEXT: v_lshl_b32_e32 v0, 0xff, v0
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, v4
+; GFX7-NEXT: v_or_b32_e32 v4, v0, v5
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v1, v4, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i8_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX10-NEXT: s_lshl_b32 s2, s4, 8
-; GFX10-NEXT: s_and_b32 s3, s4, 0xff
+; GFX10-NEXT: v_and_b32_e32 v1, 3, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v4, 2, v0
+; GFX10-NEXT: s_and_b32 s2, s4, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v3, v2, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v2, s2
+; GFX10-NEXT: v_not_b32_e32 v3, v3
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v5, v5, v3, v2
; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: s_or_b32 s2, s3, s2
-; GFX10-NEXT: v_lshlrev_b64 v[0:1], v0, 0xff
-; GFX10-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX10-NEXT: s_lshl_b32 s2, s2, 16
; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: s_or_b32 s2, s3, s2
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, v1, s2, s1
-; GFX10-NEXT: v_bfi_b32 v0, v0, s2, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v8i8_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_lshlrev_b32 v0, 3, v0
-; GFX11-NEXT: s_lshl_b32 s2, s4, 8
-; GFX11-NEXT: s_and_b32 s3, s4, 0xff
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_or_b32 s2, s3, s2
-; GFX11-NEXT: v_lshlrev_b64 v[0:1], v0, 0xff
-; GFX11-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX11-NEXT: s_lshl_b32 s2, s2, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s2, s3, s2
+; GFX11-NEXT: v_and_b32_e32 v1, 3, v0
+; GFX11-NEXT: v_lshrrev_b32_e32 v4, 2, v0
+; GFX11-NEXT: s_and_b32 s2, s4, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, v1, s2, s1
-; GFX11-NEXT: v_bfi_b32 v0, v0, s2, s0
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 3, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v4
+; GFX11-NEXT: v_lshlrev_b32_e64 v3, v2, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v2, s2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX11-NEXT: v_not_b32_e32 v3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v5, v5, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v5, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(4) %ptr
@@ -2163,101 +2154,123 @@ define amdgpu_ps void @insertelement_s_v8i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v8i8_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX9-NEXT: v_and_b32_e32 v1, 3, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX9-NEXT: s_mov_b64 s[2:3], 0xff
-; GFX9-NEXT: v_lshlrev_b64 v[1:2], v1, s[2:3]
-; GFX9-NEXT: s_mov_b32 s2, 0x4040404
-; GFX9-NEXT: v_perm_b32 v0, v0, v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xff
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, v1, v6
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v2, v3, vcc
+; GFX9-NEXT: v_not_b32_e32 v1, v1
+; GFX9-NEXT: v_and_or_b32 v1, v5, v1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX9-NEXT: global_store_dwordx2 v[3:4], v[1:2], off
+; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v8i8_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX8-NEXT: s_mov_b64 s[2:3], 0xff
-; GFX8-NEXT: v_lshlrev_b64 v[1:2], v1, s[2:3]
-; GFX8-NEXT: s_mov_b32 s2, 0x4040404
-; GFX8-NEXT: v_perm_b32 v0, v0, v0, s2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v6, 0xff
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v2, v3, vcc
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v1, v6
+; GFX8-NEXT: v_bfi_b32 v1, v1, 0, v5
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[1:2]
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
+; GFX8-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v8i8_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v1
-; GFX7-NEXT: s_mov_b64 s[2:3], 0xff
-; GFX7-NEXT: v_lshl_b64 v[0:1], s[2:3], v0
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 2, v1
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, v1, v2, s1
-; GFX7-NEXT: v_bfi_b32 v0, v0, v2, s0
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v2, v3, vcc
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v5
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v8i8_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx2 s[0:1], s[2:3], 0x0
-; GFX10-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX10-NEXT: v_perm_b32 v0, v0, v0, 0x4040404
+; GFX10-NEXT: v_and_b32_e32 v2, 3, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 2, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v2
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_not_b32_e32 v3, v4
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v1, v2, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v6, v4, v3, v0
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_lshlrev_b64 v[1:2], v1, 0xff
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX10-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX10-NEXT: global_store_dwordx2 v[3:4], v[1:2], off
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v6, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_s_v8i8_v_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 3, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v4, v2, v2, 0x4040404
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b64 v[0:1], v1, 0xff
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, v1, v4, s1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, v0, v4, s0
-; GFX11-TRUE16-NEXT: global_store_b64 v[2:3], v[0:1], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_s_v8i8_v_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_lshlrev_b32 v1, 3, v1
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0x4040404
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b64 v[1:2], v1, 0xff
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v2, v2, v0, s1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v1, v1, v0, s0
-; GFX11-FAKE16-NEXT: global_store_b64 v[3:4], v[1:2], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_s_v8i8_v_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b64 s[0:1], s[2:3], 0x0
+; GFX11-NEXT: v_and_b32_e32 v2, 3, v1
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 2, v1
+; GFX11-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_lshlrev_b32 v2, 3, v2
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v2, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, v2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v3, v4
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <8 x i8> %vec, i8 %val, i32 %idx
store <8 x i8> %insert, ptr addrspace(1) null
@@ -2268,40 +2281,46 @@ define amdgpu_ps void @insertelement_v_v8i8_s_v(ptr addrspace(1) %ptr, i8 inreg
; GFX9-LABEL: insertelement_v_v8i8_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_lshl_b32 s3, s2, 8
-; GFX9-NEXT: s_and_b32 s2, s2, 0xff
-; GFX9-NEXT: s_mov_b64 s[0:1], 0xff
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 2, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX9-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX9-NEXT: s_and_b32 s0, s2, 0xff
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX9-NEXT: s_or_b32 s2, s2, s3
-; GFX9-NEXT: v_lshlrev_b64 v[5:6], v2, s[0:1]
-; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
-; GFX9-NEXT: s_lshl_b32 s1, s2, 16
-; GFX9-NEXT: s_or_b32 s0, s0, s1
+; GFX9-NEXT: v_lshlrev_b32_e64 v7, v2, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, v2, v5
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX9-NEXT: v_not_b32_e32 v2, v2
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v6
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, v6, s0, v1
-; GFX9-NEXT: v_bfi_b32 v0, v5, s0, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX9-NEXT: v_and_or_b32 v2, v5, v2, v7
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX9-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i8_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_lshl_b32 s3, s2, 8
-; GFX8-NEXT: s_and_b32 s2, s2, 0xff
-; GFX8-NEXT: s_mov_b64 s[0:1], 0xff
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 2, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT: v_mov_b32_e32 v5, 0xff
+; GFX8-NEXT: s_and_b32 s0, s2, 0xff
; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX8-NEXT: s_or_b32 s2, s2, s3
-; GFX8-NEXT: v_lshlrev_b64 v[5:6], v2, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s0, s2, 16
-; GFX8-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX8-NEXT: v_lshlrev_b32_e64 v7, v2, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, v2, v5
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v6
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, v6, s0, v1
-; GFX8-NEXT: v_bfi_b32 v0, v5, s0, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX8-NEXT: v_bfi_b32 v2, v2, 0, v5
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v7
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -2309,62 +2328,71 @@ define amdgpu_ps void @insertelement_v_v8i8_s_v(ptr addrspace(1) %ptr, i8 inreg
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_lshl_b32 s3, s2, 8
-; GFX7-NEXT: s_and_b32 s2, s2, 0xff
-; GFX7-NEXT: s_mov_b64 s[0:1], 0xff
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 2, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xff
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX7-NEXT: s_or_b32 s2, s2, s3
-; GFX7-NEXT: v_lshl_b64 v[2:3], s[0:1], v2
-; GFX7-NEXT: s_lshl_b32 s0, s2, 16
-; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s1, s0
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
+; GFX7-NEXT: v_lshl_b32_e32 v4, s0, v2
+; GFX7-NEXT: v_lshl_b32_e32 v2, 0xff, v2
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v3
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, v3, s0, v1
-; GFX7-NEXT: v_bfi_b32 v0, v2, s0, v0
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX7-NEXT: v_bfi_b32 v2, v2, 0, v5
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i8_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX10-NEXT: s_lshl_b32 s0, s2, 8
-; GFX10-NEXT: s_and_b32 s1, s2, 0xff
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: s_or_b32 s0, s1, s0
-; GFX10-NEXT: v_lshlrev_b64 v[2:3], v2, 0xff
-; GFX10-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX10-NEXT: s_lshl_b32 s0, s0, 16
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: s_or_b32 s0, s1, s0
+; GFX10-NEXT: v_and_b32_e32 v3, 3, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 2, v2
+; GFX10-NEXT: s_and_b32 s0, s2, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v3, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v2, v3, s0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX10-NEXT: v_not_b32_e32 v3, v4
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, v3, s0, v1
-; GFX10-NEXT: v_bfi_b32 v0, v2, s0, v0
-; GFX10-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i8_s_v:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_lshlrev_b32 v2, 3, v2
-; GFX11-NEXT: s_lshl_b32 s0, s2, 8
-; GFX11-NEXT: s_and_b32 s1, s2, 0xff
-; GFX11-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-NEXT: s_or_b32 s0, s1, s0
-; GFX11-NEXT: v_lshlrev_b64 v[2:3], v2, 0xff
-; GFX11-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: s_or_b32 s0, s1, s0
+; GFX11-NEXT: v_lshrrev_b32_e32 v5, 2, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 0xff
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v5
+; GFX11-NEXT: v_and_b32_e32 v3, 3, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v3, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v2, v3, s0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v5
+; GFX11-NEXT: v_not_b32_e32 v3, v4
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, v3, s0, v1
-; GFX11-NEXT: v_bfi_b32 v0, v2, s0, v0
-; GFX11-NEXT: global_store_b64 v[4:5], v[0:1], off
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <8 x i8> %vec, i8 %val, i32 %idx
@@ -2376,80 +2404,102 @@ define amdgpu_ps void @insertelement_v_v8i8_v_s(ptr addrspace(1) %ptr, i8 %val,
; GFX9-LABEL: insertelement_v_v8i8_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b32 s0, 0x4040404
-; GFX9-NEXT: s_lshl_b32 s1, s2, 3
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX9-NEXT: s_lshl_b64 s[0:1], 0xff, s1
-; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: s_and_b32 s1, s2, 3
+; GFX9-NEXT: s_lshl_b32 s1, s1, 3
+; GFX9-NEXT: s_lshr_b32 s0, s2, 2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, s1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX9-NEXT: v_bfi_b32 v0, s0, v2, v0
-; GFX9-NEXT: global_store_dwordx2 v[3:4], v[0:1], off
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v4, v3, s1, v2
+; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i8_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b32 s0, 0x4040404
-; GFX8-NEXT: s_lshl_b32 s1, s2, 3
-; GFX8-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX8-NEXT: s_lshl_b64 s[0:1], 0xff, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, 0
-; GFX8-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-NEXT: s_and_b32 s0, s2, 3
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: s_lshr_b32 m0, s2, 2
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX8-NEXT: v_bfi_b32 v0, s0, v2, v0
-; GFX8-NEXT: flat_store_dwordx2 v[3:4], v[0:1]
+; GFX8-NEXT: v_movrels_b32_e32 v3, v0
+; GFX8-NEXT: v_bfi_b32 v3, s0, 0, v3
+; GFX8-NEXT: v_or_b32_e32 v4, v3, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_movreld_b32_e32 v0, v4
+; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v8i8_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 3
+; GFX7-NEXT: s_lshr_b32 m0, s2, 2
; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: s_lshl_b32 s0, s2, 3
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_lshl_b64 s[0:1], 0xff, s0
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT: s_lshl_b32 s0, s0, 3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, s0, v2
+; GFX7-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, s1, v2, v1
-; GFX7-NEXT: v_bfi_b32 v0, s0, v2, v0
+; GFX7-NEXT: v_movrels_b32_e32 v3, v0
+; GFX7-NEXT: v_bfi_b32 v3, s0, 0, v3
+; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_movreld_b32_e32 v0, v2
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i8_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_perm_b32 v4, v2, v2, 0x4040404
-; GFX10-NEXT: s_lshl_b32 s0, s2, 3
+; GFX10-NEXT: s_and_b32 s0, s2, 3
+; GFX10-NEXT: s_lshr_b32 m0, s2, 2
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX10-NEXT: s_not_b32 s0, s0
+; GFX10-NEXT: s_waitcnt vmcnt(0)
+; GFX10-NEXT: v_movrels_b32_e32 v3, v0
+; GFX10-NEXT: v_and_or_b32 v4, v3, s0, v2
; GFX10-NEXT: v_mov_b32_e32 v2, 0
-; GFX10-NEXT: s_lshl_b64 s[0:1], 0xff, s0
; GFX10-NEXT: v_mov_b32_e32 v3, 0
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX10-NEXT: v_bfi_b32 v0, s0, v4, v0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v4
; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_v_v8i8_v_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: v_perm_b32 v4, v2, v2, 0x4040404
-; GFX11-NEXT: s_lshl_b32 s0, s2, 3
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
-; GFX11-NEXT: s_lshl_b64 s[0:1], 0xff, s0
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 3
+; GFX11-NEXT: s_lshr_b32 m0, s2, 2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, s0, v2
+; GFX11-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_not_b32 s0, s0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_bfi_b32 v1, s1, v4, v1
-; GFX11-NEXT: v_bfi_b32 v0, s0, v4, v0
+; GFX11-NEXT: v_movrels_b32_e32 v3, v0
+; GFX11-NEXT: v_and_or_b32 v4, v3, s0, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_movreld_b32_e32 v0, v4
; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(1) %ptr
@@ -2462,104 +2512,117 @@ define amdgpu_ps void @insertelement_v_v8i8_v_v(ptr addrspace(1) %ptr, i8 %val,
; GFX9-LABEL: insertelement_v_v8i8_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX9-NEXT: s_mov_b64 s[0:1], 0xff
+; GFX9-NEXT: v_lshrrev_b32_e32 v7, 2, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX9-NEXT: v_mov_b32_e32 v6, 0xff
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX9-NEXT: s_mov_b32 s2, 0x4040404
-; GFX9-NEXT: v_lshlrev_b64 v[6:7], v3, s[0:1]
-; GFX9-NEXT: v_perm_b32 v2, v2, v2, s2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, v3, v6
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
+; GFX9-NEXT: v_not_b32_e32 v3, v3
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v7
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_bfi_b32 v1, v7, v2, v1
-; GFX9-NEXT: v_bfi_b32 v0, v6, v2, v0
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
+; GFX9-NEXT: v_and_or_b32 v2, v6, v3, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX9-NEXT: global_store_dwordx2 v[4:5], v[0:1], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v8i8_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: s_mov_b64 s[0:1], 0xff
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 2, v3
+; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX8-NEXT: v_mov_b32_e32 v6, 0xff
; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX8-NEXT: s_mov_b32 s2, 0x4040404
-; GFX8-NEXT: v_lshlrev_b64 v[6:7], v3, s[0:1]
-; GFX8-NEXT: v_perm_b32 v2, v2, v2, s2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v7
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, v3, v6
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v7
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v1, v7, v2, v1
-; GFX8-NEXT: v_bfi_b32 v0, v6, v2, v0
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v1, vcc
+; GFX8-NEXT: v_bfi_b32 v3, v3, 0, v6
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; GFX8-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v8i8_v_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
-; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v2
+; GFX7-NEXT: s_mov_b32 s6, 0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v4, 2, v3
+; GFX7-NEXT: v_and_b32_e32 v3, 3, v3
; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: s_mov_b64 s[0:1], 0xff
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v2, v4
-; GFX7-NEXT: v_lshl_b64 v[2:3], s[0:1], v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v4
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_lshl_b32_e32 v3, 0xff, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v4
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v1, v3, v4, v1
-; GFX7-NEXT: v_bfi_b32 v0, v2, v4, v0
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc
+; GFX7-NEXT: v_bfi_b32 v3, v3, 0, v5
+; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v8i8_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx2 v[0:1], v[0:1], off
-; GFX10-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX10-NEXT: v_perm_b32 v2, v2, v2, 0x4040404
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_mov_b32_e32 v6, 0
-; GFX10-NEXT: v_lshlrev_b64 v[3:4], v3, 0xff
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 2, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v6
+; GFX10-NEXT: v_lshlrev_b32_e64 v5, v4, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_not_b32_e32 v3, v5
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_bfi_b32 v1, v4, v2, v1
-; GFX10-NEXT: v_bfi_b32 v0, v3, v2, v0
-; GFX10-NEXT: global_store_dwordx2 v[5:6], v[0:1], off
+; GFX10-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX10-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX10-NEXT: v_mov_b32_e32 v2, 0
+; GFX10-NEXT: v_mov_b32_e32 v3, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX10-NEXT: global_store_dwordx2 v[2:3], v[0:1], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v8i8_v_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 3, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_perm_b32 v6, v4, v4, 0x4040404
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-TRUE16-NEXT: v_lshlrev_b64 v[2:3], v3, 0xff
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v1, v3, v6, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfi_b32 v0, v2, v6, v0
-; GFX11-TRUE16-NEXT: global_store_b64 v[4:5], v[0:1], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v8i8_v_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v6, 0 :: v_dual_lshlrev_b32 v3, 3, v3
-; GFX11-FAKE16-NEXT: v_perm_b32 v2, v2, v2, 0x4040404
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_lshlrev_b64 v[3:4], v3, 0xff
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v1, v4, v2, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfi_b32 v0, v3, v2, v0
-; GFX11-FAKE16-NEXT: global_store_b64 v[5:6], v[0:1], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v8i8_v_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-NEXT: v_and_b32_e32 v4, 3, v3
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 2, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 0, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v5, v4, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, v4, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_not_b32_e32 v3, v5
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_cndmask_b32_e32 v4, v0, v1, vcc_lo
+; GFX11-NEXT: v_and_or_b32 v4, v4, v3, v2
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v3, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc_lo
+; GFX11-NEXT: global_store_b64 v[2:3], v[0:1], off
+; GFX11-NEXT: s_endpgm
%vec = load <8 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <8 x i8> %vec, i8 %val, i32 %idx
store <8 x i8> %insert, ptr addrspace(1) null
@@ -2570,89 +2633,19 @@ define amdgpu_ps void @insertelement_s_v16i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v16i8_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: s_cmp_eq_u32 s5, 12
+; GFX9-NEXT: s_lshr_b32 m0, s5, 2
+; GFX9-NEXT: s_and_b32 s5, s5, 3
+; GFX9-NEXT: s_lshl_b32 s5, s5, 3
+; GFX9-NEXT: s_and_b32 s4, s4, 0xff
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_movrels_b32 s6, s0
+; GFX9-NEXT: s_lshl_b32 s4, s4, s5
+; GFX9-NEXT: s_lshl_b32 s5, 0xff, s5
+; GFX9-NEXT: s_andn2_b32 s5, s6, s5
+; GFX9-NEXT: s_or_b32 s4, s5, s4
+; GFX9-NEXT: s_movreld_b32 s0, s4
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cselect_b32 s6, s4, s3
-; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshr_b32 s7, s3, 8
-; GFX9-NEXT: s_cmp_eq_u32 s5, 13
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_or_b32 s6, s6, s7
-; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX9-NEXT: s_lshr_b32 s7, s3, 24
-; GFX9-NEXT: s_cmp_eq_u32 s5, 15
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_lshr_b32 s3, s3, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 14
-; GFX9-NEXT: s_cselect_b32 s3, s4, s3
-; GFX9-NEXT: s_and_b32 s3, s3, 0xff
-; GFX9-NEXT: s_or_b32 s3, s3, s7
-; GFX9-NEXT: s_lshl_b32 s3, s3, 16
-; GFX9-NEXT: s_or_b32 s3, s6, s3
-; GFX9-NEXT: s_cmp_eq_u32 s5, 8
-; GFX9-NEXT: s_cselect_b32 s6, s4, s2
-; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshr_b32 s7, s2, 8
-; GFX9-NEXT: s_cmp_eq_u32 s5, 9
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_or_b32 s6, s6, s7
-; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX9-NEXT: s_lshr_b32 s7, s2, 24
-; GFX9-NEXT: s_cmp_eq_u32 s5, 11
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 10
-; GFX9-NEXT: s_cselect_b32 s2, s4, s2
-; GFX9-NEXT: s_and_b32 s2, s2, 0xff
-; GFX9-NEXT: s_or_b32 s2, s2, s7
-; GFX9-NEXT: s_lshl_b32 s2, s2, 16
-; GFX9-NEXT: s_or_b32 s2, s6, s2
-; GFX9-NEXT: s_cmp_eq_u32 s5, 4
-; GFX9-NEXT: s_cselect_b32 s6, s4, s1
-; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshr_b32 s7, s1, 8
-; GFX9-NEXT: s_cmp_eq_u32 s5, 5
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_or_b32 s6, s6, s7
-; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX9-NEXT: s_lshr_b32 s7, s1, 24
-; GFX9-NEXT: s_cmp_eq_u32 s5, 7
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 6
-; GFX9-NEXT: s_cselect_b32 s1, s4, s1
-; GFX9-NEXT: s_and_b32 s1, s1, 0xff
-; GFX9-NEXT: s_or_b32 s1, s1, s7
-; GFX9-NEXT: s_lshl_b32 s1, s1, 16
-; GFX9-NEXT: s_or_b32 s1, s6, s1
-; GFX9-NEXT: s_cmp_eq_u32 s5, 0
-; GFX9-NEXT: s_cselect_b32 s6, s4, s0
-; GFX9-NEXT: s_and_b32 s6, s6, 0xff
-; GFX9-NEXT: s_lshr_b32 s7, s0, 8
-; GFX9-NEXT: s_cmp_eq_u32 s5, 1
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_or_b32 s6, s6, s7
-; GFX9-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX9-NEXT: s_lshr_b32 s7, s0, 24
-; GFX9-NEXT: s_cmp_eq_u32 s5, 3
-; GFX9-NEXT: s_cselect_b32 s7, s4, s7
-; GFX9-NEXT: s_lshl_b32 s7, s7, 8
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: s_cmp_eq_u32 s5, 2
-; GFX9-NEXT: s_cselect_b32 s0, s4, s0
-; GFX9-NEXT: s_and_b32 s0, s0, 0xff
-; GFX9-NEXT: s_or_b32 s0, s0, s7
-; GFX9-NEXT: s_lshl_b32 s0, s0, 16
-; GFX9-NEXT: s_or_b32 s0, s6, s0
; GFX9-NEXT: v_mov_b32_e32 v0, s0
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
@@ -2663,89 +2656,19 @@ define amdgpu_ps void @insertelement_s_v16i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX8-LABEL: insertelement_s_v16i8_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX8-NEXT: s_lshr_b32 m0, s5, 2
+; GFX8-NEXT: s_and_b32 s5, s5, 3
+; GFX8-NEXT: s_lshl_b32 s5, s5, 3
+; GFX8-NEXT: s_and_b32 s4, s4, 0xff
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_movrels_b32 s6, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, s5
+; GFX8-NEXT: s_lshl_b32 s5, 0xff, s5
+; GFX8-NEXT: s_andn2_b32 s5, s6, s5
+; GFX8-NEXT: s_or_b32 s4, s5, s4
+; GFX8-NEXT: s_movreld_b32 s0, s4
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s6, s3, 24
-; GFX8-NEXT: s_cmp_eq_u32 s5, 15
-; GFX8-NEXT: s_cselect_b32 s6, s4, s6
-; GFX8-NEXT: s_lshr_b32 s7, s3, 16
-; GFX8-NEXT: s_lshl_b32 s6, s6, 8
-; GFX8-NEXT: s_cmp_eq_u32 s5, 14
-; GFX8-NEXT: s_cselect_b32 s7, s4, s7
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 12
-; GFX8-NEXT: s_cselect_b32 s7, s4, s3
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_lshr_b32 s3, s3, 8
-; GFX8-NEXT: s_cmp_eq_u32 s5, 13
-; GFX8-NEXT: s_cselect_b32 s3, s4, s3
-; GFX8-NEXT: s_lshl_b32 s3, s3, 8
-; GFX8-NEXT: s_or_b32 s3, s7, s3
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_or_b32 s3, s3, s6
-; GFX8-NEXT: s_lshr_b32 s6, s2, 24
-; GFX8-NEXT: s_cmp_eq_u32 s5, 11
-; GFX8-NEXT: s_cselect_b32 s6, s4, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 8
-; GFX8-NEXT: s_lshr_b32 s7, s2, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 10
-; GFX8-NEXT: s_cselect_b32 s7, s4, s7
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 8
-; GFX8-NEXT: s_cselect_b32 s7, s4, s2
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_lshr_b32 s2, s2, 8
-; GFX8-NEXT: s_cmp_eq_u32 s5, 9
-; GFX8-NEXT: s_cselect_b32 s2, s4, s2
-; GFX8-NEXT: s_lshl_b32 s2, s2, 8
-; GFX8-NEXT: s_or_b32 s2, s7, s2
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s2, s2, s6
-; GFX8-NEXT: s_lshr_b32 s6, s1, 24
-; GFX8-NEXT: s_cmp_eq_u32 s5, 7
-; GFX8-NEXT: s_cselect_b32 s6, s4, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 8
-; GFX8-NEXT: s_lshr_b32 s7, s1, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 6
-; GFX8-NEXT: s_cselect_b32 s7, s4, s7
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 4
-; GFX8-NEXT: s_cselect_b32 s7, s4, s1
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_lshr_b32 s1, s1, 8
-; GFX8-NEXT: s_cmp_eq_u32 s5, 5
-; GFX8-NEXT: s_cselect_b32 s1, s4, s1
-; GFX8-NEXT: s_lshl_b32 s1, s1, 8
-; GFX8-NEXT: s_or_b32 s1, s7, s1
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s1, s6
-; GFX8-NEXT: s_lshr_b32 s6, s0, 24
-; GFX8-NEXT: s_cmp_eq_u32 s5, 3
-; GFX8-NEXT: s_cselect_b32 s6, s4, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 8
-; GFX8-NEXT: s_lshr_b32 s7, s0, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 2
-; GFX8-NEXT: s_cselect_b32 s7, s4, s7
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_cmp_eq_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s7, s4, s0
-; GFX8-NEXT: s_and_b32 s7, s7, 0xff
-; GFX8-NEXT: s_lshr_b32 s0, s0, 8
-; GFX8-NEXT: s_cmp_eq_u32 s5, 1
-; GFX8-NEXT: s_cselect_b32 s0, s4, s0
-; GFX8-NEXT: s_lshl_b32 s0, s0, 8
-; GFX8-NEXT: s_or_b32 s0, s7, s0
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s6
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s2
@@ -2756,291 +2679,69 @@ define amdgpu_ps void @insertelement_s_v16i8_s_s(ptr addrspace(4) inreg %ptr, i8
; GFX7-LABEL: insertelement_s_v16i8_s_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX7-NEXT: s_lshr_b32 m0, s5, 2
+; GFX7-NEXT: s_and_b32 s5, s5, 3
+; GFX7-NEXT: s_lshl_b32 s5, s5, 3
+; GFX7-NEXT: s_and_b32 s4, s4, 0xff
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s6, s3, 24
-; GFX7-NEXT: s_cmp_eq_u32 s5, 15
-; GFX7-NEXT: s_cselect_b32 s6, s4, s6
-; GFX7-NEXT: s_lshr_b32 s7, s3, 16
-; GFX7-NEXT: s_lshl_b32 s6, s6, 24
-; GFX7-NEXT: s_cmp_eq_u32 s5, 14
-; GFX7-NEXT: s_cselect_b32 s7, s4, s7
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshl_b32 s7, s7, 16
-; GFX7-NEXT: s_or_b32 s6, s6, s7
-; GFX7-NEXT: s_cmp_eq_u32 s5, 12
-; GFX7-NEXT: s_cselect_b32 s7, s4, s3
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshr_b32 s3, s3, 8
-; GFX7-NEXT: s_cmp_eq_u32 s5, 13
-; GFX7-NEXT: s_cselect_b32 s3, s4, s3
-; GFX7-NEXT: s_lshl_b32 s3, s3, 8
-; GFX7-NEXT: s_or_b32 s3, s7, s3
-; GFX7-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX7-NEXT: s_or_b32 s3, s3, s6
-; GFX7-NEXT: s_lshr_b32 s6, s2, 24
-; GFX7-NEXT: s_cmp_eq_u32 s5, 11
-; GFX7-NEXT: s_cselect_b32 s6, s4, s6
-; GFX7-NEXT: s_lshl_b32 s6, s6, 24
-; GFX7-NEXT: s_lshr_b32 s7, s2, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 10
-; GFX7-NEXT: s_cselect_b32 s7, s4, s7
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshl_b32 s7, s7, 16
-; GFX7-NEXT: s_or_b32 s6, s6, s7
-; GFX7-NEXT: s_cmp_eq_u32 s5, 8
-; GFX7-NEXT: s_cselect_b32 s7, s4, s2
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshr_b32 s2, s2, 8
-; GFX7-NEXT: s_cmp_eq_u32 s5, 9
-; GFX7-NEXT: s_cselect_b32 s2, s4, s2
-; GFX7-NEXT: s_lshl_b32 s2, s2, 8
-; GFX7-NEXT: s_or_b32 s2, s7, s2
-; GFX7-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX7-NEXT: s_or_b32 s2, s2, s6
-; GFX7-NEXT: s_lshr_b32 s6, s1, 24
-; GFX7-NEXT: s_cmp_eq_u32 s5, 7
-; GFX7-NEXT: s_cselect_b32 s6, s4, s6
-; GFX7-NEXT: s_lshl_b32 s6, s6, 24
-; GFX7-NEXT: s_lshr_b32 s7, s1, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 6
-; GFX7-NEXT: s_cselect_b32 s7, s4, s7
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshl_b32 s7, s7, 16
-; GFX7-NEXT: s_or_b32 s6, s6, s7
-; GFX7-NEXT: s_cmp_eq_u32 s5, 4
-; GFX7-NEXT: s_cselect_b32 s7, s4, s1
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshr_b32 s1, s1, 8
-; GFX7-NEXT: s_cmp_eq_u32 s5, 5
-; GFX7-NEXT: s_cselect_b32 s1, s4, s1
-; GFX7-NEXT: s_lshl_b32 s1, s1, 8
-; GFX7-NEXT: s_or_b32 s1, s7, s1
-; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX7-NEXT: s_or_b32 s1, s1, s6
-; GFX7-NEXT: s_lshr_b32 s6, s0, 24
-; GFX7-NEXT: s_cmp_eq_u32 s5, 3
-; GFX7-NEXT: s_cselect_b32 s6, s4, s6
-; GFX7-NEXT: s_lshl_b32 s6, s6, 24
-; GFX7-NEXT: s_lshr_b32 s7, s0, 16
-; GFX7-NEXT: s_cmp_eq_u32 s5, 2
-; GFX7-NEXT: s_cselect_b32 s7, s4, s7
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshl_b32 s7, s7, 16
-; GFX7-NEXT: s_or_b32 s6, s6, s7
-; GFX7-NEXT: s_cmp_eq_u32 s5, 0
-; GFX7-NEXT: s_cselect_b32 s7, s4, s0
-; GFX7-NEXT: s_and_b32 s7, s7, 0xff
-; GFX7-NEXT: s_lshr_b32 s0, s0, 8
-; GFX7-NEXT: s_cmp_eq_u32 s5, 1
-; GFX7-NEXT: s_cselect_b32 s0, s4, s0
-; GFX7-NEXT: s_lshl_b32 s0, s0, 8
-; GFX7-NEXT: s_or_b32 s0, s7, s0
-; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s0, s6
+; GFX7-NEXT: s_movrels_b32 s6, s0
+; GFX7-NEXT: s_lshl_b32 s4, s4, s5
+; GFX7-NEXT: s_lshl_b32 s5, 0xff, s5
+; GFX7-NEXT: s_andn2_b32 s5, s6, s5
+; GFX7-NEXT: s_or_b32 s4, s5, s4
+; GFX7-NEXT: s_movreld_b32 s0, s4
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i8_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX10-NEXT: s_cmp_eq_u32 s5, 12
+; GFX10-NEXT: s_lshr_b32 m0, s5, 2
+; GFX10-NEXT: s_and_b32 s5, s5, 3
+; GFX10-NEXT: s_and_b32 s4, s4, 0xff
+; GFX10-NEXT: s_lshl_b32 s5, s5, 3
; GFX10-NEXT: v_mov_b32_e32 v4, 0
+; GFX10-NEXT: s_lshl_b32 s7, 0xff, s5
+; GFX10-NEXT: s_lshl_b32 s4, s4, s5
; GFX10-NEXT: v_mov_b32_e32 v5, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cselect_b32 s6, s4, s3
-; GFX10-NEXT: s_lshr_b32 s7, s3, 8
-; GFX10-NEXT: s_and_b32 s6, s6, 0xff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 13
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_or_b32 s6, s6, s7
-; GFX10-NEXT: s_lshr_b32 s7, s3, 24
-; GFX10-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 15
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshr_b32 s3, s3, 16
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_cmp_eq_u32 s5, 14
-; GFX10-NEXT: s_cselect_b32 s3, s4, s3
-; GFX10-NEXT: s_and_b32 s3, s3, 0xff
-; GFX10-NEXT: s_or_b32 s3, s3, s7
-; GFX10-NEXT: s_lshl_b32 s3, s3, 16
-; GFX10-NEXT: s_or_b32 s3, s6, s3
-; GFX10-NEXT: s_cmp_eq_u32 s5, 8
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: s_cselect_b32 s6, s4, s2
-; GFX10-NEXT: s_lshr_b32 s7, s2, 8
-; GFX10-NEXT: s_and_b32 s6, s6, 0xff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 9
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_or_b32 s6, s6, s7
-; GFX10-NEXT: s_lshr_b32 s7, s2, 24
-; GFX10-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 11
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshr_b32 s2, s2, 16
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_cmp_eq_u32 s5, 10
-; GFX10-NEXT: s_cselect_b32 s2, s4, s2
-; GFX10-NEXT: s_and_b32 s2, s2, 0xff
-; GFX10-NEXT: s_or_b32 s2, s2, s7
-; GFX10-NEXT: s_lshl_b32 s2, s2, 16
-; GFX10-NEXT: s_or_b32 s2, s6, s2
-; GFX10-NEXT: s_cmp_eq_u32 s5, 4
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: s_cselect_b32 s6, s4, s1
-; GFX10-NEXT: s_lshr_b32 s7, s1, 8
-; GFX10-NEXT: s_and_b32 s6, s6, 0xff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 5
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_or_b32 s6, s6, s7
-; GFX10-NEXT: s_lshr_b32 s7, s1, 24
-; GFX10-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 7
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshr_b32 s1, s1, 16
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_cmp_eq_u32 s5, 6
-; GFX10-NEXT: s_cselect_b32 s1, s4, s1
-; GFX10-NEXT: s_and_b32 s1, s1, 0xff
-; GFX10-NEXT: s_or_b32 s1, s1, s7
-; GFX10-NEXT: s_lshl_b32 s1, s1, 16
-; GFX10-NEXT: s_or_b32 s1, s6, s1
-; GFX10-NEXT: s_cmp_eq_u32 s5, 0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: s_cselect_b32 s6, s4, s0
-; GFX10-NEXT: s_lshr_b32 s7, s0, 8
-; GFX10-NEXT: s_and_b32 s6, s6, 0xff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 1
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_or_b32 s6, s6, s7
-; GFX10-NEXT: s_lshr_b32 s7, s0, 24
-; GFX10-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX10-NEXT: s_cmp_eq_u32 s5, 3
-; GFX10-NEXT: s_cselect_b32 s7, s4, s7
-; GFX10-NEXT: s_lshr_b32 s0, s0, 16
-; GFX10-NEXT: s_lshl_b32 s7, s7, 8
-; GFX10-NEXT: s_cmp_eq_u32 s5, 2
-; GFX10-NEXT: s_cselect_b32 s0, s4, s0
-; GFX10-NEXT: s_and_b32 s0, s0, 0xff
-; GFX10-NEXT: s_or_b32 s0, s0, s7
-; GFX10-NEXT: s_lshl_b32 s0, s0, 16
-; GFX10-NEXT: s_or_b32 s0, s6, s0
+; GFX10-NEXT: s_movrels_b32 s6, s0
+; GFX10-NEXT: s_andn2_b32 s5, s6, s7
+; GFX10-NEXT: s_or_b32 s4, s5, s4
+; GFX10-NEXT: s_movreld_b32 s0, s4
; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s3
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: insertelement_s_v16i8_s_s:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-NEXT: s_cmp_eq_u32 s5, 12
+; GFX11-NEXT: s_lshr_b32 m0, s5, 2
+; GFX11-NEXT: s_and_b32 s5, s5, 3
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_lshl_b32 s5, s5, 3
; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: s_lshl_b32 s7, 0xff, s5
+; GFX11-NEXT: s_lshl_b32 s4, s4, s5
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cselect_b32 s6, s4, s3
-; GFX11-NEXT: s_lshr_b32 s7, s3, 8
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 13
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_or_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s7, s3, 24
-; GFX11-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 15
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_lshr_b32 s3, s3, 16
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_cmp_eq_u32 s5, 14
-; GFX11-NEXT: s_cselect_b32 s3, s4, s3
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s3, s3, 0xff
-; GFX11-NEXT: s_or_b32 s3, s3, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s3, s3, 16
-; GFX11-NEXT: s_or_b32 s3, s6, s3
-; GFX11-NEXT: s_cmp_eq_u32 s5, 8
-; GFX11-NEXT: v_mov_b32_e32 v3, s3
-; GFX11-NEXT: s_cselect_b32 s6, s4, s2
-; GFX11-NEXT: s_lshr_b32 s7, s2, 8
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 9
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_or_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s7, s2, 24
-; GFX11-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 11
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_lshr_b32 s2, s2, 16
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_cmp_eq_u32 s5, 10
-; GFX11-NEXT: s_cselect_b32 s2, s4, s2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s2, s2, 0xff
-; GFX11-NEXT: s_or_b32 s2, s2, s7
+; GFX11-NEXT: s_movrels_b32 s6, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s2, s2, 16
-; GFX11-NEXT: s_or_b32 s2, s6, s2
-; GFX11-NEXT: s_cmp_eq_u32 s5, 4
-; GFX11-NEXT: s_cselect_b32 s6, s4, s1
-; GFX11-NEXT: s_lshr_b32 s7, s1, 8
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 5
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_or_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s7, s1, 24
-; GFX11-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 7
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_cmp_eq_u32 s5, 6
-; GFX11-NEXT: s_cselect_b32 s1, s4, s1
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s1, s1, 0xff
-; GFX11-NEXT: s_or_b32 s1, s1, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-NEXT: s_or_b32 s1, s6, s1
-; GFX11-NEXT: s_cmp_eq_u32 s5, 0
-; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: s_cselect_b32 s6, s4, s0
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
-; GFX11-NEXT: s_and_b32 s6, s6, 0xff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 1
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_or_b32 s6, s6, s7
-; GFX11-NEXT: s_lshr_b32 s7, s0, 24
-; GFX11-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX11-NEXT: s_cmp_eq_u32 s5, 3
-; GFX11-NEXT: s_cselect_b32 s7, s4, s7
-; GFX11-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-NEXT: s_lshl_b32 s7, s7, 8
-; GFX11-NEXT: s_cmp_eq_u32 s5, 2
-; GFX11-NEXT: s_cselect_b32 s0, s4, s0
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b32 s0, s0, 0xff
-; GFX11-NEXT: s_or_b32 s0, s0, s7
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-NEXT: s_or_b32 s0, s6, s0
+; GFX11-NEXT: s_and_not1_b32 s5, s6, s7
+; GFX11-NEXT: s_or_b32 s4, s5, s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_movreld_b32 s0, s4
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(4) %ptr
@@ -3053,179 +2754,43 @@ define amdgpu_ps void @insertelement_v_v16i8_s_s(ptr addrspace(1) %ptr, i8 inreg
; GFX9-LABEL: insertelement_v_v16i8_s_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
-; GFX9-NEXT: s_cmp_eq_u32 s3, 15
+; GFX9-NEXT: s_and_b32 s1, s3, 3
+; GFX9-NEXT: s_and_b32 s2, s2, 0xff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 3
+; GFX9-NEXT: s_lshr_b32 s0, s3, 2
+; GFX9-NEXT: s_lshl_b32 s2, s2, s1
+; GFX9-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 14
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_sdwa v5, v3, v4, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 12
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 13
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 11
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 24, v2
-; GFX9-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 10
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 8
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 9
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v2
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 7
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX9-NEXT: v_lshl_or_b32 v3, v5, 16, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v10, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 6
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v11, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 4
-; GFX9-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v12, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 5
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v1
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 3
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 24, v0
-; GFX9-NEXT: v_lshl_or_b32 v2, v6, 16, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v13, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 2
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v0
-; GFX9-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v14, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 0
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s3, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_sdwa v4, v16, v4, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v1, v5, 16, v1
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v5, v0
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v6, v5, s1, v4
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v0, v6, 16, v0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i8_s_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s3, 12
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 13
+; GFX8-NEXT: s_and_b32 s0, s3, 3
+; GFX8-NEXT: s_lshr_b32 m0, s3, 2
+; GFX8-NEXT: s_and_b32 s1, s2, 0xff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: s_lshl_b32 s1, s1, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v4, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 15
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 14
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 8
-; GFX8-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX8-NEXT: v_lshlrev_b16_e32 v7, 8, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 9
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX8-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 11
-; GFX8-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v8, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 10
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v9, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 4
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v10, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 5
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 8, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v1
-; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 7
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v11, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 6
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v12, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 0
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v13, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 1
-; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v0, v4, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 3
-; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v0
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s3, 2
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v5, v14, v4, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v6, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v5, v0, v4, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_movrels_b32_e32 v4, v0
+; GFX8-NEXT: v_bfi_b32 v4, s0, 0, v4
+; GFX8-NEXT: v_or_b32_e32 v6, s1, v4
; GFX8-NEXT: v_mov_b32_e32 v4, 0
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movreld_b32_e32 v0, v6
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
@@ -3233,392 +2798,62 @@ define amdgpu_ps void @insertelement_v_v16i8_s_s(ptr addrspace(1) %ptr, i8 inreg
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_cmp_eq_u32 s3, 15
-; GFX7-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 14
+; GFX7-NEXT: s_and_b32 s0, s3, 3
+; GFX7-NEXT: s_lshr_b32 m0, s3, 2
+; GFX7-NEXT: s_and_b32 s1, s2, 0xff
+; GFX7-NEXT: s_lshl_b32 s0, s0, 3
+; GFX7-NEXT: s_lshl_b32 s1, s1, s0
+; GFX7-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 12
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 13
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v3
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 11
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 24, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 10
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 8
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v9, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 9
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 8, v2
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 7
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 24, v1
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v10, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 6
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v11, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 4
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v12, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 8, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 3
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 24, v0
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v13, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 2
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v14, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 0
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v15, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s3, 1
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 8, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v5, v6, v5
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT: v_movrels_b32_e32 v4, v0
+; GFX7-NEXT: v_bfi_b32 v4, s0, 0, v4
+; GFX7-NEXT: v_or_b32_e32 v4, s1, v4
+; GFX7-NEXT: v_movreld_b32_e32 v0, v4
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i8_s_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[0:3], v[0:1], off
-; GFX10-NEXT: s_cmp_eq_u32 s3, 15
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 14
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 12
-; GFX10-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 13
+; GFX10-NEXT: s_and_b32 s0, s3, 3
+; GFX10-NEXT: s_lshr_b32 m0, s3, 2
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
+; GFX10-NEXT: s_and_b32 s1, s2, 0xff
+; GFX10-NEXT: s_lshl_b32 s2, 0xff, s0
+; GFX10-NEXT: s_lshl_b32 s0, s1, s0
+; GFX10-NEXT: s_not_b32 s1, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 24, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 8, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 11
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 10
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 8
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 9
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 24, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, s2, s1
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 7
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 6
-; GFX10-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v10, s2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 4
-; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, s2, s4
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 5
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 3
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 8, v1
-; GFX10-NEXT: s_cselect_b32 s5, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 2
-; GFX10-NEXT: v_lshrrev_b32_e32 v15, 8, v0
-; GFX10-NEXT: s_cselect_b32 s6, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 0
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v6, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v9
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v12, s2, s4
-; GFX10-NEXT: s_cselect_b32 s4, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s3, 1
-; GFX10-NEXT: v_lshrrev_b32_e32 v13, 24, v0
-; GFX10-NEXT: s_cselect_b32 s3, -1, 0
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, s2, s3
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, s1
-; GFX10-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v13, s2, s5
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, s4
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v11, s2, s0
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v8, 8, v9
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v14, s2, s6
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_or_b32_sdwa v5, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_or_b32_sdwa v7, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX10-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX10-NEXT: v_lshl_or_b32 v2, v6, 16, v2
-; GFX10-NEXT: v_lshl_or_b32 v1, v5, 16, v1
+; GFX10-NEXT: v_movrels_b32_e32 v4, v0
+; GFX10-NEXT: v_and_or_b32 v6, v4, s1, s0
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v7, 16, v0
+; GFX10-NEXT: v_movreld_b32_e32 v0, v6
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v16i8_s_s:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 13
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 15
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 14
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 8
-; GFX11-TRUE16-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 9
-; GFX11-TRUE16-NEXT: s_cselect_b32 s7, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 11
-; GFX11-TRUE16-NEXT: s_cselect_b32 s8, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 10
-; GFX11-TRUE16-NEXT: s_cselect_b32 s9, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 4
-; GFX11-TRUE16-NEXT: s_cselect_b32 s10, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 5
-; GFX11-TRUE16-NEXT: s_cselect_b32 s11, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 7
-; GFX11-TRUE16-NEXT: s_cselect_b32 s12, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 6
-; GFX11-TRUE16-NEXT: s_cselect_b32 s13, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s14, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 1
-; GFX11-TRUE16-NEXT: s_cselect_b32 s15, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s16, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s3, 2
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, s2, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 24, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, s2, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v2.l, s2, s6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v2.h, s2, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v1.l, s2, s10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 24, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v1.h, s2, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v0.l, s2, s14
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 24, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, s2, s3
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v5.l, s2, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v6.l, s2, s4
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v3.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v7.l, s2, s7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v8.l, s2, s8
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v9.l, s2, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v10.l, s2, s12
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v13.l, s2, s15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v14.l, s2, s16
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.h, 8, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v10.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v5.l, v4.l
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v5.h, v4.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v6.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.h, v9.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v7.l, v9.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v7.h, v10.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v10.h
-; GFX11-TRUE16-NEXT: global_store_b128 v[11:12], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v16i8_s_s:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v[0:1], off
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 15
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 14
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 12
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 13
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 24, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v4, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 11
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v3, s2, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, v6, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 10
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v5, v5, s2, s1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v7, v7, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 8, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, v8, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 9
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 24, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v2, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v9, v9, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v3, v6
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, v10, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, v11, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, v5, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v7, v12, s2, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 8, v0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v9, 8, v9
-; GFX11-FAKE16-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s3, 1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 24, v0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, s2, s4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v9, v15, s2, s3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v10, v13, s2, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v11, v14, s2, s1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v9, 8, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v10
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xff, v11
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v9
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v8, v6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v10, v7
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v2
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v6, 16, v1
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v7, 16, v0
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v16i8_s_s:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b128 v[0:3], v[0:1], off
+; GFX11-NEXT: s_and_b32 s0, s3, 3
+; GFX11-NEXT: s_lshr_b32 m0, s3, 2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: s_lshl_b32 s2, 0xff, s0
+; GFX11-NEXT: s_lshl_b32 s0, s1, s0
+; GFX11-NEXT: s_not_b32 s1, s2
+; GFX11-NEXT: v_mov_b32_e32 v5, 0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_movrels_b32_e32 v4, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v6, v4, s1, s0
+; GFX11-NEXT: v_mov_b32_e32 v4, 0
+; GFX11-NEXT: v_movreld_b32_e32 v0, v6
+; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(1 ) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
store <16 x i8> %insert, ptr addrspace(1) null
@@ -3629,1167 +2864,288 @@ define amdgpu_ps void @insertelement_s_v16i8_v_s(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v16i8_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX9-NEXT: s_lshr_b32 s5, s4, 2
+; GFX9-NEXT: s_and_b32 s4, s4, 3
+; GFX9-NEXT: s_mov_b32 m0, s5
+; GFX9-NEXT: s_lshl_b32 s4, s4, 3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s5, s3, 24
-; GFX9-NEXT: s_cmp_eq_u32 s4, 15
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s5, s3, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 14
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_cmp_eq_u32 s4, 12
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s3, s3, 8
-; GFX9-NEXT: s_cmp_eq_u32 s4, 13
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v3
-; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_lshr_b32 s3, s2, 24
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: s_cmp_eq_u32 s4, 11
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 10
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: s_cmp_eq_u32 s4, 8
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT: s_movrels_b32 s6, s0
+; GFX9-NEXT: s_lshl_b32 s7, 0xff, s4
+; GFX9-NEXT: s_andn2_b32 s6, s6, s7
+; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_lshl_or_b32 v6, v0, s4, v1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s2, s2, 8
-; GFX9-NEXT: s_cmp_eq_u32 s4, 9
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_lshr_b32 s2, s1, 24
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: s_cmp_eq_u32 s4, 7
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s2, s1, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 6
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s2
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: s_cmp_eq_u32 s4, 4
-; GFX9-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 8
-; GFX9-NEXT: s_cmp_eq_u32 s4, 5
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX9-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_lshr_b32 s1, s0, 24
-; GFX9-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NEXT: s_cmp_eq_u32 s4, 3
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v4
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s1, s0, 16
-; GFX9-NEXT: s_cmp_eq_u32 s4, 2
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: s_cmp_eq_u32 s4, 0
-; GFX9-NEXT: v_or_b32_sdwa v6, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 8
-; GFX9-NEXT: s_cmp_eq_u32 s4, 1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v5, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v0, v6, 16, v0
+; GFX9-NEXT: s_set_gpr_idx_on s5, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v0, v6
+; GFX9-NEXT: s_set_gpr_idx_off
; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i8_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: s_cmp_eq_u32 s4, 12
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
+; GFX8-NEXT: s_lshr_b32 m0, s4, 2
+; GFX8-NEXT: s_and_b32 s4, s4, 3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 3
+; GFX8-NEXT: v_mov_b32_e32 v1, s4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s5, s3, 8
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: s_cmp_eq_u32 s4, 13
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s5, s3, 24
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 15
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: s_cmp_eq_u32 s4, 14
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 8
-; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s3, s2, 8
-; GFX8-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: s_cmp_eq_u32 s4, 9
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s3, s2, 24
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 11
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: s_cmp_eq_u32 s4, 10
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 4
-; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s1, 8
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: s_movrels_b32 s5, s0
+; GFX8-NEXT: s_lshl_b32 s4, 0xff, s4
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: s_andn2_b32 s4, s5, s4
+; GFX8-NEXT: v_or_b32_e32 v6, s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_cmp_eq_u32 s4, 5
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s1, 24
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 7
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: s_cmp_eq_u32 s4, 6
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 0
-; GFX8-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s0, 8
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
-; GFX8-NEXT: s_cmp_eq_u32 s4, 1
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s0, 24
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: s_cmp_eq_u32 s4, 3
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_or_b32_sdwa v6, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, s1
-; GFX8-NEXT: s_cmp_eq_u32 s4, 2
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mov_b32_e32 v4, 0
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movreld_b32_e32 v0, v6
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i8_v_s:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX7-NEXT: s_lshr_b32 m0, s4, 2
+; GFX7-NEXT: s_and_b32 s4, s4, 3
+; GFX7-NEXT: s_lshl_b32 s4, s4, 3
+; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s5, s3, 24
-; GFX7-NEXT: s_cmp_eq_u32 s4, 15
-; GFX7-NEXT: v_mov_b32_e32 v1, s5
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s5, s3, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 14
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s5
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cmp_eq_u32 s4, 12
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s3, s3, 8
-; GFX7-NEXT: s_cmp_eq_u32 s4, 13
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: s_lshr_b32 s3, s2, 24
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_cmp_eq_u32 s4, 11
-; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s3, s2, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 10
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: s_cmp_eq_u32 s4, 8
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT: s_movrels_b32 s5, s0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s4, v0
+; GFX7-NEXT: s_lshl_b32 s4, 0xff, s4
+; GFX7-NEXT: s_andn2_b32 s4, s5, s4
+; GFX7-NEXT: v_or_b32_e32 v4, s4, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s2, s2, 8
-; GFX7-NEXT: s_cmp_eq_u32 s4, 9
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX7-NEXT: s_lshr_b32 s2, s1, 24
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_cmp_eq_u32 s4, 7
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s2, s1, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 6
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: s_cmp_eq_u32 s4, 4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX7-NEXT: v_mov_b32_e32 v4, s1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s1, s1, 8
-; GFX7-NEXT: s_cmp_eq_u32 s4, 5
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX7-NEXT: s_lshr_b32 s1, s0, 24
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX7-NEXT: s_cmp_eq_u32 s4, 3
-; GFX7-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX7-NEXT: v_mov_b32_e32 v4, s1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s1, s0, 16
-; GFX7-NEXT: s_cmp_eq_u32 s4, 2
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 24, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: s_cmp_eq_u32 s4, 0
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_lshr_b32 s0, s0, 8
-; GFX7-NEXT: s_cmp_eq_u32 s4, 1
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v5, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT: v_mov_b32_e32 v3, s3
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: v_movreld_b32_e32 v0, v4
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i8_v_s:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s1, s11, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 15
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s11, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 14
-; GFX10-NEXT: v_cndmask_b32_sdwa v2, s1, v0, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 12
-; GFX10-NEXT: v_cndmask_b32_e64 v1, s2, v0, s0
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s0, s11, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 13
-; GFX10-NEXT: v_cndmask_b32_e32 v3, s11, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s0, v0, vcc_lo
-; GFX10-NEXT: s_lshr_b32 s0, s10, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 11
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v2, s0, v0, vcc_lo
-; GFX10-NEXT: s_lshr_b32 s0, s10, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 10
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 8
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s0, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s0, s10, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 9
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s10, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v4, s0, v0, vcc_lo
-; GFX10-NEXT: s_lshr_b32 s0, s9, 24
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: s_cmp_eq_u32 s4, 7
-; GFX10-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s3, s9, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 6
-; GFX10-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s0, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 4
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s2, s9, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 5
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s9, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_lshr_b32 s5, s8, 24
-; GFX10-NEXT: s_cmp_eq_u32 s4, 3
-; GFX10-NEXT: v_or_b32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v6, s2, v0, s1
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_lshr_b32 s6, s8, 16
-; GFX10-NEXT: s_cmp_eq_u32 s4, 2
-; GFX10-NEXT: v_cndmask_b32_e64 v8, s5, v0, s1
-; GFX10-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s4, 0
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_lshr_b32 s7, s8, 8
-; GFX10-NEXT: s_cmp_eq_u32 s4, 1
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s8, v0, vcc_lo
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: s_mov_b32 vcc_lo, s1
-; GFX10-NEXT: v_cndmask_b32_e64 v11, s3, v0, s0
-; GFX10-NEXT: v_cndmask_b32_sdwa v10, s7, v0, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v8
-; GFX10-NEXT: v_cndmask_b32_e64 v0, s6, v0, s2
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: v_or_b32_sdwa v8, v9, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v3, v1, 16, v3
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff, v8
-; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
-; GFX10-NEXT: v_lshl_or_b32 v1, v5, 16, v6
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX10-NEXT: s_and_b32 s5, s4, 3
+; GFX10-NEXT: s_lshr_b32 m0, s4, 2
+; GFX10-NEXT: s_lshl_b32 s4, s5, 3
+; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT: s_lshl_b32 s5, 0xff, s4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v7
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_movrels_b32 s6, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_andn2_b32 s5, s6, s5
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_lshl_or_b32 v6, v0, s4, s5
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-NEXT: v_movreld_b32_e32 v0, v6
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_s_v16i8_v_s:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 24
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 15
-; GFX11-TRUE16-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s3, 16
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, s5, v0.l, s6
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, s7, v0.l, s5
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s3, 8
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s3, v0.l, s5
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 24
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 11
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, s6, v0.l, s3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s2, 16
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, s5, v0.l, s3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 8
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, s6, v0.l, s3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s2, 8
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 9
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, s2, v0.l, s3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 24
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, s5, v0.l, s2
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 7
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 6
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, s3, v0.l, s2
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 4
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, s5, v0.l, s2
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.h, v1.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s1, v0.l, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 5
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v4.l
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 24
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s6, s0, 16
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s1, v0.l, s2
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, s3, v0.l, s5
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 8
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s4, 1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, s6, v0.l, s1
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, s0, v0.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, s3, v0.l, s1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v4.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v4.l, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v5.l, v4.h
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, 0
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v5.h, v0.l
-; GFX11-TRUE16-NEXT: global_store_b128 v[6:7], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_s_v16i8_v_s:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s11, 24
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 15
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s11, 16
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 14
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 12
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s11, 8
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 13
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, s11, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s10, 24
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 11
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s10, 16
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 10
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s10, 8
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, s10, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s9, 24
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s9, 16
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 6
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, v3, v5
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s9, 8
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 5
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v6, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, s9, v0, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s8, 24
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, s1, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s8, 16
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, s2, v0, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s8, 8
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s4, 1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v9, s8, v0, s0
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v10, s2, v0, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v11, s3, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, s5, v0, s1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v10
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v6, v8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v11
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v7, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v9
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v1, 16, v2
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v10
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v4, 16, v6
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v7
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
- %vec = load <16 x i8>, ptr addrspace(4) %ptr
- %insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
- store <16 x i8> %insert, ptr addrspace(1) null
- ret void
-}
-
-define amdgpu_ps void @insertelement_s_v16i8_s_v(ptr addrspace(4) inreg %ptr, i8 inreg %val, i32 %idx) {
-; GFX9-LABEL: insertelement_s_v16i8_s_v:
+; GFX11-LABEL: insertelement_s_v16i8_v_s:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
+; GFX11-NEXT: s_and_b32 s5, s4, 3
+; GFX11-NEXT: s_lshr_b32 m0, s4, 2
+; GFX11-NEXT: s_lshl_b32 s4, s5, 3
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT: s_lshl_b32 s5, 0xff, s4
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_movrels_b32 s6, s0
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_and_not1_b32 s5, s6, s5
+; GFX11-NEXT: v_mov_b32_e32 v3, s3
+; GFX11-NEXT: v_lshl_or_b32 v6, v0, s4, s5
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_movreld_b32_e32 v0, v6
+; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX11-NEXT: s_endpgm
+ %vec = load <16 x i8>, ptr addrspace(4) %ptr
+ %insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
+ store <16 x i8> %insert, ptr addrspace(1) null
+ ret void
+}
+
+define amdgpu_ps void @insertelement_s_v16i8_s_v(ptr addrspace(4) inreg %ptr, i8 inreg %val, i32 %idx) {
+; GFX9-LABEL: insertelement_s_v16i8_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 2, v0
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX9-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s5, s3, 24
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_lshr_b32 s4, s3, 16
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v1, v4, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
-; GFX9-NEXT: s_lshr_b32 s3, s3, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v3
-; GFX9-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: s_lshr_b32 s3, s2, 24
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GFX9-NEXT: s_lshr_b32 s2, s2, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX9-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NEXT: s_lshr_b32 s2, s1, 24
-; GFX9-NEXT: v_lshl_or_b32 v2, v1, 16, v2
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX9-NEXT: s_lshr_b32 s2, s1, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX9-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX9-NEXT: s_lshr_b32 s1, s1, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX9-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: s_lshr_b32 s1, s0, 24
-; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX9-NEXT: s_lshr_b32 s1, s0, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX9-NEXT: v_or_b32_sdwa v6, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v0, v4, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v4, 0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v5
+; GFX9-NEXT: s_and_b32 s4, s4, 0xff
+; GFX9-NEXT: v_mov_b32_e32 v8, 0xff
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v3, s[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v5
+; GFX9-NEXT: v_lshlrev_b32_e64 v7, v0, s4
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v0, v8
+; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[2:3]
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_and_or_b32 v7, v6, v0, v7
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v1, v7, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v3, v7, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GFX9-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX9-NEXT: v_mov_b32_e32 v6, 0
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, v7, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[5:6], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i8_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 2, v0
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX8-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: s_lshr_b32 s4, s3, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: s_lshr_b32 s4, s3, 24
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GFX8-NEXT: s_lshr_b32 s3, s2, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: s_lshr_b32 s3, s2, 24
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX8-NEXT: s_lshr_b32 s2, s1, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: s_lshr_b32 s2, s1, 24
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX8-NEXT: s_lshr_b32 s1, s0, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX8-NEXT: s_lshr_b32 s1, s0, 24
-; GFX8-NEXT: v_or_b32_sdwa v6, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX8-NEXT: v_mov_b32_e32 v7, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v7, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v4, s3
+; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v5
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v3, s[0:1]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v5
+; GFX8-NEXT: s_and_b32 s4, s4, 0xff
+; GFX8-NEXT: v_mov_b32_e32 v8, 0xff
+; GFX8-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[2:3]
+; GFX8-NEXT: v_lshlrev_b32_e64 v7, v0, s4
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v0, v8
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v6
+; GFX8-NEXT: v_or_b32_e32 v7, v0, v7
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v1, v7, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v7, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v3, v7, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v5, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v6, 0
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, v7, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[5:6], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i8_s_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX7-NEXT: v_mov_b32_e32 v4, s4
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v5, 2, v0
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v5
+; GFX7-NEXT: v_and_b32_e32 v0, 3, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s4, s3, 24
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: s_lshr_b32 s4, s3, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s4
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v0
-; GFX7-NEXT: s_lshr_b32 s3, s3, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_lshr_b32 s3, s2, 24
-; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v0
-; GFX7-NEXT: s_lshr_b32 s3, s2, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v0
-; GFX7-NEXT: s_lshr_b32 s2, s2, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: s_lshr_b32 s2, s1, 24
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v0
-; GFX7-NEXT: s_lshr_b32 s2, s1, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v0
-; GFX7-NEXT: s_lshr_b32 s1, s1, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: s_lshr_b32 s1, s0, 24
-; GFX7-NEXT: v_or_b32_e32 v1, v5, v1
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v0
-; GFX7-NEXT: s_lshr_b32 s1, s0, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX7-NEXT: s_lshr_b32 s0, s0, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v4, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v4, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT: v_mov_b32_e32 v2, s1
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: v_mov_b32_e32 v4, s3
+; GFX7-NEXT: v_mov_b32_e32 v3, s2
+; GFX7-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v5
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v3, s[0:1]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v5
+; GFX7-NEXT: s_and_b32 s4, s4, 0xff
+; GFX7-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[2:3]
+; GFX7-NEXT: v_lshl_b32_e32 v7, s4, v0
+; GFX7-NEXT: v_lshl_b32_e32 v0, 0xff, v0
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, v6
+; GFX7-NEXT: v_or_b32_e32 v6, v0, v7
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v1, v6, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v2, v6, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v3, v6, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v4, v6, s[2:3]
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i8_s_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x0
-; GFX10-NEXT: v_mov_b32_e32 v1, s4
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 12, v0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX10-NEXT: v_lshrrev_b32_e32 v6, 2, v0
+; GFX10-NEXT: v_and_b32_e32 v4, 3, v0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX10-NEXT: v_lshlrev_b32_e64 v7, v4, 0xff
+; GFX10-NEXT: v_not_b32_e32 v7, v7
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s1, s11, 24
-; GFX10-NEXT: s_lshr_b32 s3, s11, 8
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, s1, v1, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v0
-; GFX10-NEXT: s_lshr_b32 s2, s11, 16
-; GFX10-NEXT: s_lshr_b32 s4, s10, 24
-; GFX10-NEXT: v_cndmask_b32_e64 v2, s11, v1, s0
-; GFX10-NEXT: s_lshr_b32 s5, s10, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s3, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 8, v0
-; GFX10-NEXT: s_lshr_b32 s7, s9, 8
-; GFX10-NEXT: s_lshr_b32 s6, s9, 24
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s2, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v3, s10, v1, s0
-; GFX10-NEXT: s_lshr_b32 s0, s10, 8
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s4, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v0
-; GFX10-NEXT: s_lshr_b32 s1, s8, 24
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s5, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v0
-; GFX10-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s0, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT: s_lshr_b32 s0, s9, 16
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v9
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s7, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s6, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s9, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s0, v1, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 3, v0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v6, s1, v1, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v0
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v10, s8, v1, s0
-; GFX10-NEXT: s_lshr_b32 s0, s8, 8
-; GFX10-NEXT: v_cndmask_b32_sdwa v11, s0, v1, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT: s_lshr_b32 s0, s8, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s0, v1, vcc_lo
-; GFX10-NEXT: v_or_b32_sdwa v1, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v4, 16, v2
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX10-NEXT: s_and_b32 s1, s4, 0xff
+; GFX10-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX10-NEXT: v_lshlrev_b32_e64 v4, v4, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
+; GFX10-NEXT: v_and_or_b32 v7, v5, v7, v4
; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v7, 16, v10
-; GFX10-NEXT: v_lshl_or_b32 v1, v8, 16, v5
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v6
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_s_v16i8_s_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 12, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 14, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 13, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s3, 11, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 10, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s5, 8, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s6, 9, v0
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v7, 0 :: v_dual_mov_b32 v8, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s11, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s11, 16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s11, v1.l, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s11, 8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, s7, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, s12, v1.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, s1, v1.l, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s10, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s10, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s10, 8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, s11, v1.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, s13, v1.l, s4
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, s10, v1.l, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, s14, v1.l, s6
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v1.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v5.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 4, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 6, v0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s9, 24
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, s2, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s9, v1.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s9, 16
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 0, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, s3, v1.l, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s9, 8
-; GFX11-TRUE16-NEXT: v_and_b16 v5.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s0, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 2, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 1, v0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s8, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s8, 24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, s4, v1.l, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s8, 8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, s3, v1.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, s8, v1.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, s0, v1.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v4.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v5.l, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v5.h, v6.h
-; GFX11-TRUE16-NEXT: global_store_b128 v[7:8], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_s_v16i8_s_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v0
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, s3, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s3, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, s2, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s2, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, s4, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, s5, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, s3, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, s6, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v8, s7, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, s2, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s1, 24
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, v8, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, s2, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s1, 8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, s1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v8, s2, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, s1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v10, s1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v9, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v10
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v11, s0, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v4, 16, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v12, s1, v1, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v10
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_cndmask_b32 v0, s0, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v11
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v11, 8, v12
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v11
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v8
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_s_v16i8_s_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
+; GFX11-NEXT: v_lshrrev_b32_e32 v6, 2, v0
+; GFX11-NEXT: v_and_b32_e32 v4, 3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v6
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX11-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
+; GFX11-NEXT: v_lshlrev_b32_e32 v4, 3, v4
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v6
+; GFX11-NEXT: s_and_b32 s1, s4, 0xff
+; GFX11-NEXT: v_cndmask_b32_e32 v5, v0, v1, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v6
+; GFX11-NEXT: v_lshlrev_b32_e64 v7, v4, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v4, v4, s1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v6
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v2, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v7, v7
+; GFX11-NEXT: v_cndmask_b32_e64 v5, v5, v3, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v7, v5, v7, v4
+; GFX11-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, v7, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v7, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: global_store_b128 v[4:5], v[0:3], off
+; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
store <16 x i8> %insert, ptr addrspace(1) null
@@ -4800,536 +3156,163 @@ define amdgpu_ps void @insertelement_s_v16i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX9-LABEL: insertelement_s_v16i8_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v6, 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX9-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s3, 24
-; GFX9-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-NEXT: s_lshr_b32 s4, s3, 16
-; GFX9-NEXT: v_cndmask_b32_sdwa v2, v2, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 14, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v1
-; GFX9-NEXT: s_lshr_b32 s3, s3, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v4, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 13, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v4
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: s_lshr_b32 s3, s2, 24
-; GFX9-NEXT: v_lshl_or_b32 v4, v2, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 11, v1
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 10, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v1
-; GFX9-NEXT: s_lshr_b32 s2, s2, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 9, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX9-NEXT: v_or_b32_sdwa v3, v3, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NEXT: s_lshr_b32 s2, s1, 24
-; GFX9-NEXT: v_lshl_or_b32 v3, v2, 16, v3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX9-NEXT: s_lshr_b32 s2, s1, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v5, s2
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v2, 8, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX9-NEXT: s_lshr_b32 s1, s1, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX9-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX9-NEXT: s_lshr_b32 s1, s0, 24
-; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v5
-; GFX9-NEXT: v_mov_b32_e32 v5, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX9-NEXT: s_lshr_b32 s1, s0, 16
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v6, s1
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX9-NEXT: v_or_b32_sdwa v7, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_mov_b32_e32 v5, s0
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX9-NEXT: s_lshr_b32 s0, s0, 8
-; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v1, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v5, 0
+; GFX9-NEXT: v_mov_b32_e32 v3, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v5, s3
+; GFX9-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v6
+; GFX9-NEXT: v_mov_b32_e32 v8, 0xff
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v4, s[0:1]
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v6
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, v1, v8
+; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[2:3]
+; GFX9-NEXT: v_not_b32_e32 v1, v1
+; GFX9-NEXT: v_and_or_b32 v8, v7, v1, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v2, v8, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v3, v8, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v4, v8, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v6, 0
-; GFX9-NEXT: v_lshl_or_b32 v1, v7, 16, v0
-; GFX9-NEXT: global_store_dwordx4 v[5:6], v[1:4], off
+; GFX9-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v5, v8, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[6:7], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_s_v16i8_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 12, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX8-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: s_lshr_b32 s4, s3, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v3
-; GFX8-NEXT: s_lshr_b32 s4, s3, 24
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 15, v1
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v4, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 14, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v4, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 8, v1
-; GFX8-NEXT: s_lshr_b32 s3, s2, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 9, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v3
-; GFX8-NEXT: s_lshr_b32 s3, s2, 24
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 11, v1
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 10, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX8-NEXT: s_lshr_b32 s2, s1, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: s_lshr_b32 s2, s1, 24
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX8-NEXT: s_lshr_b32 s1, s0, 8
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX8-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX8-NEXT: s_lshr_b32 s1, s0, 24
-; GFX8-NEXT: v_or_b32_sdwa v7, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, s1
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v6, s0
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v6
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v4, s[0:1]
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v6
+; GFX8-NEXT: v_mov_b32_e32 v8, 0xff
+; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[2:3]
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, v1, v8
+; GFX8-NEXT: v_bfi_b32 v1, v1, 0, v7
+; GFX8-NEXT: v_or_b32_e32 v8, v1, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v2, v8, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v8, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v4, v8, s[0:1]
; GFX8-NEXT: v_mov_b32_e32 v6, 0
-; GFX8-NEXT: v_or_b32_sdwa v1, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[5:6], v[1:4]
+; GFX8-NEXT: v_mov_b32_e32 v7, 0
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v5, v8, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_s_v16i8_v_v:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v6, 2, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s4, s3, 24
-; GFX7-NEXT: v_mov_b32_e32 v2, s4
-; GFX7-NEXT: s_lshr_b32 s4, s3, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s4
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v1
-; GFX7-NEXT: s_lshr_b32 s3, s3, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v4, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT: s_lshr_b32 s3, s2, 24
-; GFX7-NEXT: v_or_b32_e32 v4, v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v1
-; GFX7-NEXT: s_lshr_b32 s3, s2, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v1
-; GFX7-NEXT: s_lshr_b32 s2, s2, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v3, v3, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT: s_lshr_b32 s2, s1, 24
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v1
-; GFX7-NEXT: s_lshr_b32 s2, s1, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v5, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v1
-; GFX7-NEXT: s_lshr_b32 s1, s1, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: s_lshr_b32 s1, s0, 24
-; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX7-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v1
-; GFX7-NEXT: s_lshr_b32 s1, s0, 16
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v6, s1
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX7-NEXT: v_mov_b32_e32 v6, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v1
-; GFX7-NEXT: s_lshr_b32 s0, s0, 8
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_mov_b32_e32 v7, s0
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v7, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v1, v0, v5
+; GFX7-NEXT: v_mov_b32_e32 v3, s1
+; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: v_mov_b32_e32 v5, s3
+; GFX7-NEXT: v_mov_b32_e32 v4, s2
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v2, v3, vcc
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v6
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v4, s[0:1]
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v6
+; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[2:3]
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v1, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v6
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v2, v7, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v3, v7, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v4, v7, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v5, v7, s[2:3]
; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: buffer_store_dwordx4 v[1:4], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_s_v16i8_v_v:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v1
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 12, v1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[2:3], 0x0
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 2, v1
+; GFX10-NEXT: v_and_b32_e32 v5, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 3, v5
+; GFX10-NEXT: v_lshlrev_b32_e64 v8, v5, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_not_b32_e32 v5, v8
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s1, s7, 24
-; GFX10-NEXT: s_lshr_b32 s3, s7, 8
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, s1, v0, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v1
-; GFX10-NEXT: s_lshr_b32 s2, s7, 16
-; GFX10-NEXT: v_cndmask_b32_e64 v2, s7, v0, s0
-; GFX10-NEXT: s_lshr_b32 s7, s6, 24
-; GFX10-NEXT: s_lshr_b32 s8, s6, 16
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s3, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v1
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 8, v1
-; GFX10-NEXT: s_lshr_b32 s9, s5, 8
-; GFX10-NEXT: s_lshr_b32 s1, s4, 24
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s2, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v1
-; GFX10-NEXT: v_cndmask_b32_e64 v3, s6, v0, s0
-; GFX10-NEXT: s_lshr_b32 s0, s6, 8
-; GFX10-NEXT: v_or_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: s_lshr_b32 s6, s5, 24
-; GFX10-NEXT: v_cndmask_b32_e32 v7, s7, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v1
-; GFX10-NEXT: v_or_b32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s8, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v1
-; GFX10-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s0, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v1
-; GFX10-NEXT: s_lshr_b32 s0, s5, 16
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v9
-; GFX10-NEXT: v_cndmask_b32_e32 v5, s9, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v1
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v8, s6, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v1
-; GFX10-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX10-NEXT: v_cndmask_b32_e32 v6, s5, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v1
-; GFX10-NEXT: v_or_b32_sdwa v5, v6, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v9, s0, v0, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 3, v1
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v6, s1, v0, s0
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v1
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v10, s4, v0, s0
-; GFX10-NEXT: s_lshr_b32 s0, s4, 8
-; GFX10-NEXT: v_cndmask_b32_sdwa v11, s0, v0, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v1
-; GFX10-NEXT: s_lshr_b32 s0, s4, 16
-; GFX10-NEXT: v_or_b32_sdwa v1, v10, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v0, s0, v0, vcc_lo
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v4, 16, v2
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v7, 16, v10
-; GFX10-NEXT: v_lshl_or_b32 v1, v8, 16, v5
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
+; GFX10-NEXT: v_mov_b32_e32 v4, s3
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
+; GFX10-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
+; GFX10-NEXT: v_and_or_b32 v8, v6, v5, v0
; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v6
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_mov_b32_e32 v6, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
+; GFX10-NEXT: global_store_dwordx4 v[5:6], v[0:3], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_s_v16i8_v_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b128 s[8:11], s[2:3], 0x0
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 12, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 14, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s3, 11, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 10, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 13, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s5, 8, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s6, 9, v1
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v7, 0
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s7, s11, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s12, s11, 16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, s11, v0.l, s1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s11, 8
-; GFX11-TRUE16-NEXT: s_lshr_b32 s11, s10, 24
-; GFX11-TRUE16-NEXT: s_lshr_b32 s13, s10, 16
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, s7, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, s12, v0.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, s11, v0.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, s13, v0.l, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, s1, v0.l, s2
-; GFX11-TRUE16-NEXT: s_lshr_b32 s14, s10, 8
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v3.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v1
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, s10, v0.l, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, s14, v0.l, s6
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 6, v1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s9, 24
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v2.h, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v4.l, v4.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, s2, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s9, 16
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.h, v3.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v2.l, 8, v5.h
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 4, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, s3, v0.l, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s9, 8
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 1, v1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, s0, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, s9, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v1
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 0, v1
-; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s8, 16
-; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s8, 24
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, s4, v0.l, s0
-; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s8, 8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, s3, v0.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, s8, v0.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, s0, v0.l, s2
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.l, 8, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v5.h, 0xff, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.h, v4.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v0.h, v5.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v6.h, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v6, 0
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v5.h, v0.l
-; GFX11-TRUE16-NEXT: global_store_b128 v[6:7], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_s_v16i8_v_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v1
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v2, s3, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s3, 24
-; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 8
-; GFX11-FAKE16-NEXT: s_lshr_b32 s6, s2, 24
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v3, s2, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, s4, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, s5, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, s3, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, s6, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v8, s7, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, s2, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s1, 24
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, v8, v7
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, s2, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s1, 8
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v3, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, s1, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v8, s2, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, s1, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 24
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v10, s1, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s0, 8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v9, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v10
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v11, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v4, 16, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v12, s1, v0, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v11
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v5, 16, v10
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v11, 8, v12
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, s0, v0, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v11
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v6, 16, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v8
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_s_v16i8_v_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[2:3], 0x0
+; GFX11-NEXT: v_lshrrev_b32_e32 v7, 2, v1
+; GFX11-NEXT: v_and_b32_e32 v5, 3, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v7
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v3, s2 :: v_dual_mov_b32 v2, s1
+; GFX11-NEXT: v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v4, s3
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 3, v5
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v7
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v7
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-NEXT: v_cndmask_b32_e32 v6, v1, v2, vcc_lo
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v7
+; GFX11-NEXT: v_lshlrev_b32_e64 v8, v5, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v5, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v3, s0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_not_b32_e32 v5, v8
+; GFX11-NEXT: v_cndmask_b32_e64 v6, v6, v4, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v8, v6, v5, v0
+; GFX11-NEXT: v_dual_mov_b32 v5, 0 :: v_dual_mov_b32 v6, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v1, v8, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v2, v8, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v3, v8, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v4, v8, s1
+; GFX11-NEXT: global_store_b128 v[5:6], v[0:3], off
+; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(4) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
store <16 x i8> %insert, ptr addrspace(1) null
@@ -5340,475 +3323,151 @@ define amdgpu_ps void @insertelement_v_v16i8_s_v(ptr addrspace(1) %ptr, i8 inreg
; GFX9-LABEL: insertelement_v_v16i8_s_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v2
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 14, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 13, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 11, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 10, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 9, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 6, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 5, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[16:17], 3, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[18:19], 2, v2
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 2, v2
+; GFX9-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX9-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX9-NEXT: s_and_b32 s0, s2, 0xff
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_e64 v9, v2, s0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX9-NEXT: v_mov_b32_e32 v7, 0
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v6, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v6, v0, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v5, v0, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v11, v4, v0, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v0, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v6
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v6, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v8, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v5, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[6:7]
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v5
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_lshl_or_b32 v5, v1, 16, v6
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v10, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[10:11]
-; GFX9-NEXT: v_or_b32_sdwa v1, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v6, v9, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v4, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 8, v4
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[14:15]
-; GFX9-NEXT: v_lshl_or_b32 v4, v1, 16, v6
-; GFX9-NEXT: v_cndmask_b32_sdwa v6, v12, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[16:17]
-; GFX9-NEXT: v_or_b32_sdwa v1, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v3, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[18:19]
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v3
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[20:21]
-; GFX9-NEXT: v_or_b32_sdwa v6, v11, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v13, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v6
-; GFX9-NEXT: v_or_b32_sdwa v6, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_lshl_or_b32 v2, v6, 16, v2
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[2:3]
+; GFX9-NEXT: v_and_or_b32 v9, v2, v0, v9
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v6, v9, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i8_s_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 15, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 12, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 14, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 9, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 11, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 10, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 5, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 7, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 6, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[18:19], 3, v2
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[20:21], 2, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 2, v2
+; GFX8-NEXT: v_and_b32_e32 v2, 3, v2
+; GFX8-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX8-NEXT: s_and_b32 s0, s2, 0xff
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v2
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_e64 v9, v2, s0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX8-NEXT: v_mov_b32_e32 v7, 0
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX8-NEXT: v_cndmask_b32_e64 v1, v6, v0, s[22:23]
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v7, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v6, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 8, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v5
-; GFX8-NEXT: v_cndmask_b32_sdwa v6, v6, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e64 v8, v5, v0, s[22:23]
-; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v9, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[6:7]
-; GFX8-NEXT: v_or_b32_sdwa v6, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v5, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 4, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 8, v4
-; GFX8-NEXT: v_cndmask_b32_sdwa v5, v5, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[10:11]
-; GFX8-NEXT: v_cndmask_b32_e64 v10, v4, v0, s[22:23]
-; GFX8-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v11, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX8-NEXT: v_or_b32_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v10, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v4, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[14:15]
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 8, v3
-; GFX8-NEXT: v_cndmask_b32_sdwa v4, v4, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[16:17]
-; GFX8-NEXT: v_or_b32_sdwa v4, v4, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v12, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[18:19]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 0, v2
-; GFX8-NEXT: v_or_b32_sdwa v4, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v3, v0, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[20:21]
-; GFX8-NEXT: v_cndmask_b32_e64 v2, v3, v0, s[22:23]
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v3, v0, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v3, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
+; GFX8-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v5, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v6, s[2:3]
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v2
+; GFX8-NEXT: v_or_b32_e32 v9, v0, v9
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v3, v9, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v5, v9, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v6, v9, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[7:8], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v16i8_s_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s6, 0
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
-; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v2
-; GFX7-NEXT: s_mov_b64 s[4:5], 0
-; GFX7-NEXT: s_mov_b32 s6, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, 0xf000
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 2, v2
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v2
+; GFX7-NEXT: s_and_b32 s0, s2, 0xff
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_lshl_b32_e32 v2, s0, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v0
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v0
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: s_mov_b32 s10, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v6, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v11, v5, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v14, v4, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v17, v3, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 8, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 24, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v12, v12, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v13, v13, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v4, v4, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v15, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v16, v16, v0, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v2
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX7-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX7-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX7-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX7-NEXT: v_and_b32_e32 v17, 0xff, v17
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v6
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v10
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_and_b32_e32 v10, 0xff, v13
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_and_b32_e32 v13, 0xff, v16
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 24, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v9, 24, v12
-; GFX7-NEXT: v_lshlrev_b32_e32 v12, 24, v15
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v5, v11, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
-; GFX7-NEXT: v_or_b32_e32 v4, v14, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v13
-; GFX7-NEXT: v_or_b32_e32 v0, v17, v0
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v7
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_or_b32_e32 v7, v9, v8
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX7-NEXT: v_or_b32_e32 v8, v12, v10
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v5, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v4, v7
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; GFX7-NEXT: v_cndmask_b32_e32 v7, v3, v4, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v7, v7, v6, s[2:3]
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v7
+; GFX7-NEXT: v_or_b32_e32 v7, v1, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v3, v7, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v4, v7, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v5, v7, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v6, v7, s[2:3]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i8_s_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v2
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 2, v2
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v2
+; GFX10-NEXT: s_and_b32 s1, s2, 0xff
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v7, v0, 0xff
+; GFX10-NEXT: v_lshlrev_b32_e64 v0, v0, s1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: v_not_b32_e32 v7, v7
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v6, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v0, 24, v6
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 8, v6
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v5
-; GFX10-NEXT: v_cndmask_b32_e64 v10, v5, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v13, v4, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v16, v3, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v2
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 8, v3
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v2
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v2
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v2
-; GFX10-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v12, v12, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v4, v4, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX10-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX10-NEXT: v_cndmask_b32_e64 v14, v14, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT: v_or_b32_sdwa v4, v13, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v15, v15, s2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v3, s2, vcc_lo
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v6
-; GFX10-NEXT: v_lshlrev_b16 v6, 8, v8
-; GFX10-NEXT: v_lshlrev_b16 v8, 8, v11
-; GFX10-NEXT: v_lshlrev_b16 v11, 8, v14
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_or_b32_sdwa v1, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v3, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v6, v9, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v7, v12, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v2, v16, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v8, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v3
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v4
-; GFX10-NEXT: v_and_b32_e32 v11, 0xffff, v2
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshl_or_b32 v3, v0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v6, 16, v9
-; GFX10-NEXT: v_lshl_or_b32 v1, v7, 16, v10
-; GFX10-NEXT: v_lshl_or_b32 v0, v8, 16, v11
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v5, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s1
+; GFX10-NEXT: v_and_or_b32 v9, v2, v7, v0
+; GFX10-NEXT: v_mov_b32_e32 v7, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v3, v9, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v5, v9, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v9, s1
+; GFX10-NEXT: global_store_dwordx4 v[7:8], v[0:3], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v16i8_s_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s3, 14, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s8, 4, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s11, 6, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 13, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 15, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 8, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s5, 9, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s6, 11, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s7, 10, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s9, 5, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s10, 7, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s12, 0, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s13, 1, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s14, 3, v2
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s15, 2, v2
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, s2, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, s2, s3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v4.l, s2, s8
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 24, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v4.h, s2, s11
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, s2, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, s2, s7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v3.l, s2, s12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, s2, s15
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v7.l, s2, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v8.l, s2, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v6.l, s2, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v11.l, s2, s6
-; GFX11-TRUE16-NEXT: v_and_b16 v6.l, 0xff, v2.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v12.l, s2, s9
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.h, v13.l, s2, s10
-; GFX11-TRUE16-NEXT: v_and_b16 v7.l, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v14.l, s2, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v15.l, s2, s14
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v2.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.h, 8, v6.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v7.h
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v0.h, v3.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v4.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.h, v5.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.l, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v7.l, v6.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v7.h
-; GFX11-TRUE16-NEXT: global_store_b128 v[9:10], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v16i8_s_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v2
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v7, v6, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 24, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v10, v5, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v13, v4, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v16, v3, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xff, v10
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xff, v13
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v16, 0xff, v16
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, v6, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v8, v8, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v9, v9, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v5, v5, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v11, v11, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v12, v12, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v4, v4, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v14, v14, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v13, v4
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v15, v15, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v3, s2, vcc_lo
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v9, 8, v11
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xff, v12
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v12, 8, v14
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xff, v15
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v7, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v10, v5
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v16, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v8, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v11, v9
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v14, v12
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_and_b32 v11, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v0, 16, v1
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v6, 16, v9
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v7, 16, v10
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v8, 16, v11
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v16i8_s_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 2, v2
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v2
+; GFX11-NEXT: s_and_b32 s1, s2, 0xff
+; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_lshlrev_b32_e64 v7, v0, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e64 v0, v0, s1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_not_b32_e32 v7, v7
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc_lo
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v5, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v2, v6, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_and_or_b32 v9, v2, v7, v0
+; GFX11-NEXT: v_mov_b32_e32 v7, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v3, v9, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v4, v9, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v5, v9, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v6, v9, s1
+; GFX11-NEXT: global_store_b128 v[7:8], v[0:3], off
+; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
store <16 x i8> %insert, ptr addrspace(1) null
@@ -5819,177 +3478,41 @@ define amdgpu_ps void @insertelement_v_v16i8_v_s(ptr addrspace(1) %ptr, i8 %val,
; GFX9-LABEL: insertelement_v_v16i8_v_s:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX9-NEXT: s_cmp_eq_u32 s2, 15
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 14
+; GFX9-NEXT: s_and_b32 s1, s2, 3
+; GFX9-NEXT: s_lshl_b32 s1, s1, 3
+; GFX9-NEXT: s_lshr_b32 s0, s2, 2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, s1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: s_lshl_b32 s1, 0xff, s1
+; GFX9-NEXT: s_not_b32 s1, s1
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v6, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 12
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 13
-; GFX9-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 11
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 24, v5
-; GFX9-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 10
-; GFX9-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 8
-; GFX9-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v7
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 9
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v5
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v5, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 7
-; GFX9-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v10, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 6
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v11, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 4
-; GFX9-NEXT: v_or_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v12, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 5
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v4, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 3
-; GFX9-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; GFX9-NEXT: v_lshl_or_b32 v4, v6, 16, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v0, v13, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 2
-; GFX9-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX9-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX9-NEXT: v_cndmask_b32_e32 v6, v14, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 0
-; GFX9-NEXT: v_or_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_lshlrev_b16_e32 v6, 8, v6
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v15, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: s_cmp_eq_u32 s2, 1
-; GFX9-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_e32 v7, v3, v2, vcc
-; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX9-NEXT: v_lshl_or_b32 v3, v1, 16, v0
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v16, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; GFX9-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: v_and_or_b32 v2, v1, s1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_lshl_or_b32 v2, v6, 16, v2
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[2:5], off
+; GFX9-NEXT: s_set_gpr_idx_on s0, gpr_idx(DST)
+; GFX9-NEXT: v_mov_b32_e32 v3, v2
+; GFX9-NEXT: s_set_gpr_idx_off
+; GFX9-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i8_v_s:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[3:6], v[0:1]
-; GFX8-NEXT: s_cmp_eq_u32 s2, 12
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 13
+; GFX8-NEXT: s_and_b32 s0, s2, 3
+; GFX8-NEXT: s_lshr_b32 m0, s2, 2
+; GFX8-NEXT: s_lshl_b32 s0, s0, 3
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v6, v2, vcc
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v6
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 15
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 14
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 8
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v7, 8, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 9
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 8, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 24, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v5
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 11
-; GFX8-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v8, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 10
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v9, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 4
-; GFX8-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v5, v10, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 5
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 8, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v4
-; GFX8-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 7
-; GFX8-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v11, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 6
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v12, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 0
-; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v13, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 1
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v7, v3, v2, vcc
-; GFX8-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 3
-; GFX8-NEXT: v_lshrrev_b32_e32 v14, 8, v3
-; GFX8-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_eq_u32 s2, 2
-; GFX8-NEXT: v_or_b32_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v14, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v7, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v3, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v3, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_movrels_b32_e32 v1, v3
+; GFX8-NEXT: v_bfi_b32 v1, s0, 0, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v1, v0
; GFX8-NEXT: v_mov_b32_e32 v0, 0
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v3, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_movreld_b32_e32 v3, v2
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[3:6]
; GFX8-NEXT: s_endpgm
;
@@ -5997,388 +3520,62 @@ define amdgpu_ps void @insertelement_v_v16i8_v_s(ptr addrspace(1) %ptr, i8 %val,
; GFX7: ; %bb.0:
; GFX7-NEXT: s_mov_b32 s6, 0
; GFX7-NEXT: s_mov_b32 s7, 0xf000
-; GFX7-NEXT: s_mov_b32 s4, s6
-; GFX7-NEXT: s_mov_b32 s5, s6
+; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: buffer_load_dwordx4 v[3:6], v[0:1], s[4:7], 0 addr64
-; GFX7-NEXT: s_cmp_eq_u32 s2, 15
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 14
+; GFX7-NEXT: s_and_b32 s0, s2, 3
+; GFX7-NEXT: s_lshr_b32 m0, s2, 2
+; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX7-NEXT: s_lshl_b32 s0, 0xff, s0
; GFX7-NEXT: s_mov_b64 s[4:5], 0
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v6
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 12
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 13
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 11
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 24, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 10
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v8, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 8
-; GFX7-NEXT: v_or_b32_e32 v1, v6, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v9, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 9
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 8, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v5, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 7
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; GFX7-NEXT: v_or_b32_e32 v5, v1, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v10, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 6
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v8
-; GFX7-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v11, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v12, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 5
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v4, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 3
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v1, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v13, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 2
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v8
-; GFX7-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v14, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 0
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v15, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: s_cmp_eq_u32 s2, 1
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v3, v2, vcc
-; GFX7-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v8
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v6, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v1, v0
-; GFX7-NEXT: buffer_store_dwordx4 v[2:5], off, s[4:7], 0
+; GFX7-NEXT: v_movrels_b32_e32 v1, v3
+; GFX7-NEXT: v_bfi_b32 v1, s0, 0, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-NEXT: v_movreld_b32_e32 v3, v0
+; GFX7-NEXT: buffer_store_dwordx4 v[3:6], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i8_v_s:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[3:6], v[0:1], off
-; GFX10-NEXT: s_cmp_eq_u32 s2, 15
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 14
+; GFX10-NEXT: s_and_b32 s0, s2, 3
+; GFX10-NEXT: s_lshr_b32 m0, s2, 2
+; GFX10-NEXT: s_lshl_b32 s0, s0, 3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX10-NEXT: s_not_b32 s0, s0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v6, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 12
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 13
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 24, v5
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v2, s0
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 11
-; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 10
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v8, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 8
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v5
-; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v2, s0
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 9
-; GFX10-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 7
-; GFX10-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; GFX10-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v7
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v8
-; GFX10-NEXT: v_cndmask_b32_e32 v8, v10, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 6
-; GFX10-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX10-NEXT: s_cselect_b32 s0, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 4
-; GFX10-NEXT: v_cndmask_b32_e32 v10, v11, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 5
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 3
-; GFX10-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; GFX10-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v6, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v8
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v13, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 2
-; GFX10-NEXT: v_lshlrev_b16 v8, 8, v10
-; GFX10-NEXT: s_cselect_b32 s1, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 0
-; GFX10-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e32 v7, v14, v2, vcc_lo
-; GFX10-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX10-NEXT: s_cmp_eq_u32 s2, 1
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v3
-; GFX10-NEXT: s_cselect_b32 s2, -1, 0
-; GFX10-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v2, vcc_lo
-; GFX10-NEXT: s_mov_b32 vcc_lo, s2
-; GFX10-NEXT: v_lshlrev_b16 v9, 8, v9
-; GFX10-NEXT: v_cndmask_b32_sdwa v10, v10, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_e64 v11, v12, v2, s0
-; GFX10-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v15, v2, s1
-; GFX10-NEXT: v_or_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_or_b32_sdwa v3, v3, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX10-NEXT: v_or_b32_sdwa v8, v11, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX10-NEXT: v_or_b32_sdwa v7, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v3
-; GFX10-NEXT: v_lshl_or_b32 v3, v0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v2, v6, 16, v5
-; GFX10-NEXT: v_lshl_or_b32 v1, v8, 16, v4
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_lshl_or_b32 v0, v7, 16, v9
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_movrels_b32_e32 v0, v3
+; GFX10-NEXT: v_and_or_b32 v2, v0, s0, v1
+; GFX10-NEXT: v_mov_b32_e32 v0, 0
+; GFX10-NEXT: v_mov_b32_e32 v1, 0
+; GFX10-NEXT: v_movreld_b32_e32 v3, v2
+; GFX10-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v16i8_v_s:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 12
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 13
-; GFX11-TRUE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 15
-; GFX11-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 14
-; GFX11-TRUE16-NEXT: s_cselect_b32 s4, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 8
-; GFX11-TRUE16-NEXT: s_cselect_b32 s5, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 9
-; GFX11-TRUE16-NEXT: s_cselect_b32 s6, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 11
-; GFX11-TRUE16-NEXT: s_cselect_b32 s7, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 10
-; GFX11-TRUE16-NEXT: s_cselect_b32 s8, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 4
-; GFX11-TRUE16-NEXT: s_cselect_b32 s9, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 5
-; GFX11-TRUE16-NEXT: s_cselect_b32 s10, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 7
-; GFX11-TRUE16-NEXT: s_cselect_b32 s11, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 6
-; GFX11-TRUE16-NEXT: s_cselect_b32 s12, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-TRUE16-NEXT: s_cselect_b32 s13, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 1
-; GFX11-TRUE16-NEXT: s_cselect_b32 s14, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 3
-; GFX11-TRUE16-NEXT: s_cselect_b32 s15, -1, 0
-; GFX11-TRUE16-NEXT: s_cmp_eq_u32 s2, 2
-; GFX11-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v6.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 24, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v6.h, v2.l, s4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 24, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v4.l, v2.l, s9
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 24, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 8, v3
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v5.l, v2.l, s5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v5.h, v2.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v2.l, s12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v3.l, v2.l, s13
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v3.h, v2.l, s2
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v7.l, v2.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v8.l, v2.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v6.l, v2.l, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v11.l, v2.l, s7
-; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v12.l, v2.l, s10
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v13.l, v2.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.h, v14.l, v2.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v15.l, v2.l, s15
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_and_b16 v4.h, 0xff, v4.h
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.l, 8, v3.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v6.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.h, 8, v7.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v3.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v0.h, v3.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.h, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.h, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v4.l, v7.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v4.h, v7.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v8.l, v8.h
-; GFX11-TRUE16-NEXT: global_store_b128 v[9:10], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v16i8_v_s:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b128 v[3:6], v[0:1], off
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 15
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 14
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 12
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 13
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 24, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v6, v6, v2, s1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, v2, s0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 11
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v7, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 10
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v8, v8, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 8
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 8, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 9
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 24, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 16, v4
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v10, v10, v2 :: v_dual_and_b32 v5, 0xff, v5
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v6, v7
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v11, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, v12, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v2, vcc_lo
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v1, v8
-; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v8, v13, v2 :: v_dual_and_b32 v9, 0xff, v9
-; GFX11-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 2
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 8, v3
-; GFX11-FAKE16-NEXT: s_cselect_b32 s0, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v10
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: s_cmp_eq_u32 s2, 1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v14, 24, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v3, v3, v2, s1
-; GFX11-FAKE16-NEXT: s_cselect_b32 s1, -1, 0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, v5, v10
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v10, v16, v2, s1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v11, v14, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v2, v15, v2, s0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v10, 8, v10
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v4, v8
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v8, 8, v11
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v3, v10
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v9, v7
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v2, v8
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v0, 16, v6
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v1, 16, v5
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v7, 16, v4
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v5, 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v8, 16, v9
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v16i8_v_s:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b128 v[3:6], v[0:1], off
+; GFX11-NEXT: v_and_b32_e32 v0, 0xff, v2
+; GFX11-NEXT: s_and_b32 s0, s2, 3
+; GFX11-NEXT: s_lshr_b32 m0, s2, 2
+; GFX11-NEXT: s_lshl_b32 s0, s0, 3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
+; GFX11-NEXT: s_lshl_b32 s0, 0xff, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_not_b32 s0, s0
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_movrels_b32_e32 v1, v3
+; GFX11-NEXT: v_and_or_b32 v2, v1, s0, v0
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_movreld_b32_e32 v3, v2
+; GFX11-NEXT: global_store_b128 v[0:1], v[3:6], off
+; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
store <16 x i8> %insert, ptr addrspace(1) null
@@ -6389,467 +3586,147 @@ define amdgpu_ps void @insertelement_v_v16i8_v_v(ptr addrspace(1) %ptr, i8 %val,
; GFX9-LABEL: insertelement_v_v16i8_v_v:
; GFX9: ; %bb.0:
; GFX9-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 15, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 14, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 13, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 11, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[6:7], 10, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[8:9], 9, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[10:11], 7, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[12:13], 6, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[14:15], 5, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[16:17], 3, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[18:19], 2, v3
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[20:21], 1, v3
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 2, v3
+; GFX9-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX9-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX9-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, v3, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX9-NEXT: v_not_b32_e32 v0, v0
+; GFX9-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX9-NEXT: v_mov_b32_e32 v8, 0
+; GFX9-NEXT: v_mov_b32_e32 v9, 0
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v7, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 12, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 8, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v11, v5, v2, vcc
-; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v2, vcc
-; GFX9-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX9-NEXT: v_lshrrev_b32_e32 v8, 8, v7
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v7, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX9-NEXT: v_or_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v8, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX9-NEXT: v_or_b32_sdwa v1, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v6, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[6:7]
-; GFX9-NEXT: v_lshrrev_b32_e32 v10, 8, v6
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX9-NEXT: v_lshl_or_b32 v6, v0, 16, v1
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v10, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[10:11]
-; GFX9-NEXT: v_or_b32_sdwa v0, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v5, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX9-NEXT: v_lshrrev_b32_e32 v12, 8, v5
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[14:15]
-; GFX9-NEXT: v_lshl_or_b32 v5, v0, 16, v1
-; GFX9-NEXT: v_cndmask_b32_sdwa v1, v12, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[16:17]
-; GFX9-NEXT: v_or_b32_sdwa v0, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v1, v11, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_cndmask_b32_sdwa v7, v4, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[18:19]
-; GFX9-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_cndmask_b32_sdwa v8, v4, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: s_mov_b64 vcc, s[20:21]
-; GFX9-NEXT: v_lshl_or_b32 v4, v0, 16, v1
-; GFX9-NEXT: v_cndmask_b32_sdwa v0, v13, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_lshl_or_b32 v3, v7, 16, v2
-; GFX9-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
+; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GFX9-NEXT: v_and_or_b32 v3, v3, v0, v2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v4, v3, s[4:5]
+; GFX9-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX9-NEXT: v_cndmask_b32_e64 v2, v6, v3, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX9-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
; GFX9-NEXT: s_endpgm
;
; GFX8-LABEL: insertelement_v_v16i8_v_v:
; GFX8: ; %bb.0:
; GFX8-NEXT: flat_load_dwordx4 v[4:7], v[0:1]
-; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 13, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 15, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 12, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 14, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 9, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[6:7], 11, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[8:9], 10, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[10:11], 5, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[12:13], 7, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[14:15], 6, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[16:17], 1, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[18:19], 3, v3
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[20:21], 2, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 2, v3
+; GFX8-NEXT: v_and_b32_e32 v3, 3, v3
+; GFX8-NEXT: v_mov_b32_e32 v0, 0xff
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 3, v3
+; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, v3, v0
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v1
+; GFX8-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v1
+; GFX8-NEXT: v_mov_b32_e32 v8, 0
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v7
-; GFX8-NEXT: v_cndmask_b32_e64 v0, v7, v2, s[22:23]
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v1, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v7, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[2:3]
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 8, v6
-; GFX8-NEXT: v_cndmask_b32_sdwa v7, v7, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX8-NEXT: v_or_b32_sdwa v1, v7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v9, v9, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[6:7]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 8, v3
-; GFX8-NEXT: v_or_b32_sdwa v7, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v6, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[8:9]
-; GFX8-NEXT: v_cndmask_b32_e64 v8, v6, v2, s[22:23]
-; GFX8-NEXT: v_lshrrev_b32_e32 v11, 8, v5
-; GFX8-NEXT: v_cndmask_b32_sdwa v6, v6, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[10:11]
-; GFX8-NEXT: v_or_b32_sdwa v0, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v6, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v8, v11, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[12:13]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 4, v3
-; GFX8-NEXT: v_or_b32_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v5, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[14:15]
-; GFX8-NEXT: v_cndmask_b32_e64 v10, v5, v2, s[22:23]
-; GFX8-NEXT: v_lshrrev_b32_e32 v12, 8, v4
-; GFX8-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[16:17]
-; GFX8-NEXT: v_or_b32_sdwa v0, v10, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v8, v12, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[18:19]
-; GFX8-NEXT: v_cmp_eq_u32_e64 s[22:23], 0, v3
-; GFX8-NEXT: v_or_b32_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_cndmask_b32_sdwa v0, v4, v2, vcc dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX8-NEXT: s_mov_b64 vcc, s[20:21]
-; GFX8-NEXT: v_cndmask_b32_e64 v3, v4, v2, s[22:23]
-; GFX8-NEXT: v_cndmask_b32_sdwa v1, v4, v2, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v0, 0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_or_b32_sdwa v4, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, v3
+; GFX8-NEXT: v_or_b32_e32 v3, v0, v2
+; GFX8-NEXT: v_cndmask_b32_e64 v0, v4, v3, s[4:5]
+; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX8-NEXT: v_cndmask_b32_e64 v2, v6, v3, s[0:1]
+; GFX8-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: insertelement_v_v16i8_v_v:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_mov_b32 s2, 0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s0, s2
-; GFX7-NEXT: s_mov_b32 s1, s2
-; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[0:3], 0 addr64
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 12, v3
-; GFX7-NEXT: s_mov_b64 s[0:1], 0
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s10, 0
+; GFX7-NEXT: s_mov_b32 s11, 0xf000
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: buffer_load_dwordx4 v[4:7], v[0:1], s[8:11], 0 addr64
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 2, v3
+; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 3, v3
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 2, v0
+; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 3, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[2:3], 3, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, v1, v2
+; GFX7-NEXT: v_lshl_b32_e32 v1, 0xff, v1
+; GFX7-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0
+; GFX7-NEXT: s_mov_b64 s[8:9], 0
+; GFX7-NEXT: s_mov_b32 s10, -1
; GFX7-NEXT: s_waitcnt vmcnt(0)
-; GFX7-NEXT: v_cndmask_b32_e32 v8, v7, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 8, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v11, v6, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 4, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v14, v5, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 24, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v17, v4, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 15, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 14, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v7
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 13, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v9, 24, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v7, v7, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 11, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 10, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 8, v6
-; GFX7-NEXT: v_cndmask_b32_e32 v10, v10, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 9, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 24, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 7, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v12, v12, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 6, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_cndmask_b32_e32 v13, v13, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 5, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 24, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 3, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v16, 16, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v15, v15, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 2, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX7-NEXT: v_cndmask_b32_e32 v16, v16, v2, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 1, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX7-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX7-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX7-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX7-NEXT: v_and_b32_e32 v17, 0xff, v17
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v7
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v10
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX7-NEXT: v_and_b32_e32 v10, 0xff, v13
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX7-NEXT: v_and_b32_e32 v13, 0xff, v16
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 24, v9
-; GFX7-NEXT: v_lshlrev_b32_e32 v9, 24, v12
-; GFX7-NEXT: v_lshlrev_b32_e32 v12, 24, v15
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v3, v8, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v6, v11, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v10
-; GFX7-NEXT: v_or_b32_e32 v5, v14, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v10, 16, v13
-; GFX7-NEXT: v_or_b32_e32 v2, v17, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v7
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_or_b32_e32 v7, v9, v8
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_or_b32_e32 v8, v12, v10
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v1, v0
-; GFX7-NEXT: v_or_b32_e32 v2, v6, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v5, v7
-; GFX7-NEXT: v_or_b32_e32 v0, v9, v8
-; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX7-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v6, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v3, v7, s[2:3]
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, v3
+; GFX7-NEXT: v_or_b32_e32 v3, v1, v2
+; GFX7-NEXT: v_cndmask_b32_e64 v0, v4, v3, s[4:5]
+; GFX7-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc
+; GFX7-NEXT: v_cndmask_b32_e64 v2, v6, v3, s[0:1]
+; GFX7-NEXT: v_cndmask_b32_e64 v3, v7, v3, s[2:3]
+; GFX7-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: insertelement_v_v16i8_v_v:
; GFX10: ; %bb.0:
; GFX10-NEXT: global_load_dwordx4 v[4:7], v[0:1], off
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 14, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 13, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 11, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s3, 10, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s4, 9, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s5, 7, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s6, 6, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s7, 5, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s8, 3, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s9, 2, v3
-; GFX10-NEXT: v_cmp_eq_u32_e64 s10, 1, v3
+; GFX10-NEXT: v_lshrrev_b32_e32 v1, 2, v3
+; GFX10-NEXT: v_and_b32_e32 v0, 3, v3
+; GFX10-NEXT: v_mov_b32_e32 v9, 0
+; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX10-NEXT: v_lshlrev_b32_e64 v8, v0, 0xff
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_not_b32_e32 v2, v8
+; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_cndmask_b32_sdwa v0, v7, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v8, 8, v7
-; GFX10-NEXT: v_lshrrev_b32_e32 v10, 8, v6
-; GFX10-NEXT: v_lshrrev_b32_e32 v12, 8, v5
-; GFX10-NEXT: v_lshrrev_b32_e32 v13, 8, v4
-; GFX10-NEXT: v_cndmask_b32_e32 v1, v7, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v9, v6, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v11, v5, v2, vcc_lo
-; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v2, vcc_lo
-; GFX10-NEXT: s_mov_b32 vcc_lo, s0
-; GFX10-NEXT: v_cndmask_b32_sdwa v7, v7, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s1
-; GFX10-NEXT: v_cndmask_b32_sdwa v8, v8, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s2
-; GFX10-NEXT: v_or_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_sdwa v7, v6, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s3
-; GFX10-NEXT: v_or_b32_sdwa v1, v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_sdwa v6, v6, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s4
-; GFX10-NEXT: v_cndmask_b32_sdwa v8, v10, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s5
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_cndmask_b32_sdwa v10, v5, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s6
-; GFX10-NEXT: v_or_b32_sdwa v8, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_sdwa v5, v5, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s7
-; GFX10-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_sdwa v12, v12, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s8
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffff, v8
-; GFX10-NEXT: v_cndmask_b32_sdwa v14, v4, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s9
-; GFX10-NEXT: v_or_b32_sdwa v9, v11, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_sdwa v4, v4, v2, vcc_lo dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: s_mov_b32 vcc_lo, s10
-; GFX10-NEXT: v_or_b32_sdwa v5, v5, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_cndmask_b32_sdwa v2, v13, v2, vcc_lo dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_e32 v8, 0xffff, v9
-; GFX10-NEXT: v_or_b32_sdwa v9, v4, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_mov_b32_e32 v4, 0
-; GFX10-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX10-NEXT: v_lshl_or_b32 v3, v0, 16, v1
-; GFX10-NEXT: v_lshl_or_b32 v1, v5, 16, v8
-; GFX10-NEXT: v_mov_b32_e32 v5, 0
-; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v2
-; GFX10-NEXT: v_lshl_or_b32 v2, v6, 16, v7
-; GFX10-NEXT: v_lshl_or_b32 v0, v9, 16, v10
-; GFX10-NEXT: global_store_dwordx4 v[4:5], v[0:3], off
+; GFX10-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v6, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX10-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v4, v3, s2
+; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v3, s0
+; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s1
+; GFX10-NEXT: global_store_dwordx4 v[8:9], v[0:3], off
; GFX10-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: insertelement_v_v16i8_v_v:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s2, 14, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s7, 4, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s10, 6, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s0, 13, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s1, 15, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s3, 8, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s4, 9, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s5, 11, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s6, 10, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s8, 5, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s9, 7, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s11, 0, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s12, 1, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s13, 3, v3
-; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e64 s14, 2, v3
-; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, 0 :: v_dual_mov_b32 v11, 0
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.l, v7.l, v2.l, vcc_lo
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 8, v7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 24, v7
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v0.h, v7.h, v2.l, s2
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 8, v6
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v12, 24, v6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v5.l, v2.l, s7
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v5
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 24, v5
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v5.h, v2.l, s10
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 8, v4
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 24, v4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.l, v6.l, v2.l, s3
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v1.h, v6.h, v2.l, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.h, v4.l, v2.l, s11
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.l, v4.h, v2.l, s14
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v4.h, v8.l, v2.l, s0
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.l, v9.l, v2.l, s1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v5.h, v7.l, v2.l, s4
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v6.l, v12.l, v2.l, s5
-; GFX11-TRUE16-NEXT: v_and_b16 v6.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.h, v13.l, v2.l, s8
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v7.l, v14.l, v2.l, s9
-; GFX11-TRUE16-NEXT: v_and_b16 v7.h, 0xff, v3.l
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, v15.l, v2.l, s12
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, v16.l, v2.l, s13
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v1.h, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v8.l, 0xff, v3.h
-; GFX11-TRUE16-NEXT: v_and_b16 v4.l, 0xff, v4.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v3.h, 8, v4.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v4.h, 8, v5.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.l, 8, v5.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v5.h, 8, v6.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v6.l, 8, v2.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v7.l, 8, v7.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v8.h, 8, v3.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v9.l, 8, v2.l
-; GFX11-TRUE16-NEXT: v_or_b16 v3.l, v0.l, v3.h
-; GFX11-TRUE16-NEXT: v_or_b16 v3.h, v0.h, v4.h
-; GFX11-TRUE16-NEXT: v_or_b16 v2.l, v1.l, v5.l
-; GFX11-TRUE16-NEXT: v_or_b16 v2.h, v1.h, v5.h
-; GFX11-TRUE16-NEXT: v_or_b16 v1.l, v6.h, v6.l
-; GFX11-TRUE16-NEXT: v_or_b16 v1.h, v7.h, v7.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v8.l, v8.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v4.l, v9.l
-; GFX11-TRUE16-NEXT: global_store_b128 v[10:11], v[0:3], off
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: insertelement_v_v16i8_v_v:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: global_load_b128 v[4:7], v[0:1], off
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 12, v3
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v8, v7, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 8, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 8, v7
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 24, v6
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v11, v6, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 4, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 16, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v12, 24, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v14, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v5
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 24, v4
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v17, v4, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 15, v3
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 14, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 13, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v7, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 11, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v9, v9, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 10, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v10, v10, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 9, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 7, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v12, v12, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 6, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v13, v13, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 5, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v5, v5, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 3, v3
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v5, 8, v5
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v15, v15, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 2, v3
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v16, v16, v2, vcc_lo
-; GFX11-FAKE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v7
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v10
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xff, v13
-; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v2, v4, v2 :: v_dual_and_b32 v17, 0xff, v17
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v8, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v4, 8, v9
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v9, 8, v12
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v6, 8, v6
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v12, 8, v15
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v13, 0xff, v16
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v7, v7, v4
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, v14, v5
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, v11, v6
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v17, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v6, v10, v9
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, v13, v12
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v10, 0xffff, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff, v3
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_and_b32 v11, 0xffff, v2
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v5, 0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v0, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v7, 16, v9
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v1, v6, 16, v10
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v8, 16, v11
-; GFX11-FAKE16-NEXT: global_store_b128 v[4:5], v[0:3], off
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: insertelement_v_v16i8_v_v:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: global_load_b128 v[4:7], v[0:1], off
+; GFX11-NEXT: v_lshrrev_b32_e32 v1, 2, v3
+; GFX11-NEXT: v_mov_b32_e32 v9, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 1, v1
+; GFX11-NEXT: v_and_b32_e32 v0, 3, v3
+; GFX11-NEXT: v_cmp_eq_u32_e64 s0, 2, v1
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT: v_cmp_eq_u32_e64 s1, 3, v1
+; GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0, v1
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_dual_cndmask_b32 v3, v4, v5 :: v_dual_lshlrev_b32 v0, 3, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v6, s0
+; GFX11-NEXT: v_lshlrev_b32_e64 v8, v0, 0xff
+; GFX11-NEXT: v_lshlrev_b32_e32 v0, v0, v2
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v3, v7, s1
+; GFX11-NEXT: v_not_b32_e32 v2, v8
+; GFX11-NEXT: v_mov_b32_e32 v8, 0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_and_or_b32 v3, v3, v2, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v4, v3, s2
+; GFX11-NEXT: v_cndmask_b32_e32 v1, v5, v3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v6, v3, s0
+; GFX11-NEXT: v_cndmask_b32_e64 v3, v7, v3, s1
+; GFX11-NEXT: global_store_b128 v[8:9], v[0:3], off
+; GFX11-NEXT: s_endpgm
%vec = load <16 x i8>, ptr addrspace(1) %ptr
%insert = insertelement <16 x i8> %vec, i8 %val, i32 %idx
store <16 x i8> %insert, ptr addrspace(1) null
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
index e946f458ba35b..0c6853fe4ae66 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract-vector-elt.mir
@@ -252,9 +252,9 @@ body: |
; CHECK-LABEL: name: extract_vector_elt_0_v2i16_i32
; CHECK: [[DEF:%[0-9]+]]:_(<2 x s16>) = G_IMPLICIT_DEF
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[DEF]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[DEF]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = G_IMPLICIT_DEF
%1:_(s32) = G_CONSTANT i32 0
%2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -438,8 +438,8 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C]](s32)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[SHL]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[SHL]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<4 x s8>) = G_BITCAST %0
@@ -460,8 +460,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(<4 x s8>) = G_BITCAST %0
%2:_(s32) = G_CONSTANT i32 0
@@ -482,8 +482,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(<4 x s8>) = G_BITCAST %0
%2:_(s32) = G_CONSTANT i32 1
@@ -504,8 +504,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(<4 x s8>) = G_BITCAST %0
%2:_(s32) = G_CONSTANT i32 2
@@ -526,8 +526,8 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(<4 x s8>) = G_BITCAST %0
%2:_(s32) = G_CONSTANT i32 3
@@ -550,16 +550,16 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[LSHR]](s32)
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x i32>), [[LSHR]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EVEC]], [[SHL]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[EVEC]], [[SHL]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR1]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr2
%2:_(<8 x s8>) = G_BITCAST %0
@@ -581,10 +581,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 0
%2:_(<8 x s8>) = G_BITCAST %0
@@ -604,11 +604,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 1
%2:_(<8 x s8>) = G_BITCAST %0
@@ -628,11 +628,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 3
%2:_(<8 x s8>) = G_BITCAST %0
@@ -652,11 +652,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[UV1]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 4
%2:_(<8 x s8>) = G_BITCAST %0
@@ -676,11 +676,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[UV1]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 5
%2:_(<8 x s8>) = G_BITCAST %0
@@ -700,11 +700,11 @@ body: |
; CHECK: liveins: $vgpr0_vgpr1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[UV1]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY1]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = G_CONSTANT i32 7
%2:_(<8 x s8>) = G_BITCAST %0
@@ -725,13 +725,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[SHL]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[SHL]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -751,9 +751,9 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(s32) = G_CONSTANT i32 0
%2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -772,10 +772,10 @@ body: |
; CHECK: liveins: $vgpr0
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(<2 x s16>) = COPY $vgpr0
%1:_(s32) = G_CONSTANT i32 1
%2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -883,15 +883,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<2 x s32>), [[LSHR]](s32)
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[BITCAST]](<2 x i32>), [[LSHR]](s32)
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EVEC]], [[SHL]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[EVEC]], [[SHL]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR1]](i32)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr2
%2:_(s16) = G_EXTRACT_VECTOR_ELT %0, %1
@@ -1370,8 +1370,8 @@ body: |
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[SHL]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY]], [[SHL]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(<32 x s1>) = G_BITCAST %0
@@ -1393,12 +1393,12 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr3
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[COPY]](<3 x s32>), [[LSHR]](s32)
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[COPY]](<3 x s32>), [[LSHR]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[EVEC]], [[SHL]](s32)
- ; CHECK-NEXT: $vgpr0 = COPY [[LSHR1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[EVEC]], [[SHL]](s32)
+ ; CHECK-NEXT: $vgpr0 = COPY [[LSHR1]](i32)
%0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
%1:_(<12 x s8>) = G_BITCAST %0
%2:_(s32) = COPY $vgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir
index 8d22656246b1a..a1f217ee76263 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert-vector-elt.mir
@@ -797,15 +797,15 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[SHL]](s32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[SHL]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C2]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[XOR]]
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
- ; CHECK-NEXT: $vgpr0 = COPY [[OR]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[SHL]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[SHL]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i32) = G_XOR [[SHL2]], [[C2]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[XOR]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
+ ; CHECK-NEXT: $vgpr0 = COPY [[OR]](i32)
%0:_(s32) = COPY $vgpr0
%1:_(s32) = COPY $vgpr1
%2:_(s32) = COPY $vgpr2
@@ -828,25 +828,25 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](s64)
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[UV]](s32), [[UV1]](s32)
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[COPY]](s64)
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[UV]](i32), [[UV1]](i32)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
- ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x s32>), [[LSHR]](s32)
+ ; CHECK-NEXT: [[EVEC:%[0-9]+]]:_(i32) = G_EXTRACT_VECTOR_ELT [[BUILD_VECTOR]](<2 x i32>), [[LSHR]](s32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[SHL]](s32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[SHL]](s32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
- ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s32) = G_XOR [[SHL2]], [[C3]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[EVEC]], [[XOR]]
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
- ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<2 x s32>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[OR]](s32), [[LSHR]](s32)
- ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[IVEC]](<2 x s32>)
- ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV2]](s32), [[UV3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C2]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[SHL]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[SHL]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 -1
+ ; CHECK-NEXT: [[XOR:%[0-9]+]]:_(i32) = G_XOR [[SHL2]], [[C3]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[EVEC]], [[XOR]]
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
+ ; CHECK-NEXT: [[IVEC:%[0-9]+]]:_(<2 x i32>) = G_INSERT_VECTOR_ELT [[BUILD_VECTOR]], [[OR]](i32), [[LSHR]](s32)
+ ; CHECK-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[IVEC]](<2 x i32>)
+ ; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[UV2]](i32), [[UV3]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
%0:_(s64) = COPY $vgpr0_vgpr1
%1:_(s32) = COPY $vgpr2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
index c59257c57d9cf..d761f2cd8a97e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.dim.a16.ll
@@ -20,9 +20,9 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -48,9 +48,9 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -76,9 +76,9 @@ define amdgpu_ps <4 x float> @load_1d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -110,13 +110,13 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -141,13 +141,13 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -172,13 +172,13 @@ define amdgpu_ps <4 x float> @load_2d(<8 x i32> inreg %rsrc, <2 x i16> %coords)
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -211,16 +211,16 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -249,16 +249,16 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -286,16 +286,16 @@ define amdgpu_ps <4 x float> @load_3d(<8 x i32> inreg %rsrc, <2 x i16> %coords_l
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -331,16 +331,16 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -369,16 +369,16 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -406,16 +406,16 @@ define amdgpu_ps <4 x float> @load_cube(<8 x i32> inreg %rsrc, <2 x i16> %coords
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -450,13 +450,13 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -481,13 +481,13 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -512,13 +512,13 @@ define amdgpu_ps <4 x float> @load_1darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1darray), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -551,16 +551,16 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -589,16 +589,16 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -626,16 +626,16 @@ define amdgpu_ps <4 x float> @load_2darray(<8 x i32> inreg %rsrc, <2 x i16> %coo
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -671,16 +671,16 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -709,16 +709,16 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -746,16 +746,16 @@ define amdgpu_ps <4 x float> @load_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -791,19 +791,19 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>)
@@ -831,19 +831,19 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -870,19 +870,19 @@ define amdgpu_ps <4 x float> @load_2darraymsaa(<8 x i32> inreg %rsrc, <2 x i16>
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -917,13 +917,13 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -948,13 +948,13 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -979,13 +979,13 @@ define amdgpu_ps <4 x float> @load_mip_1d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32), [[UV2:%[0-9]+]]:_(f32), [[UV3:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<4 x f32>)
@@ -1018,16 +1018,16 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1056,16 +1056,16 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1093,16 +1093,16 @@ define amdgpu_ps <4 x float> @load_mip_2d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1138,19 +1138,19 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>)
@@ -1178,19 +1178,19 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -1217,19 +1217,19 @@ define amdgpu_ps <4 x float> @load_mip_3d(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.3d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -1265,19 +1265,19 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>)
@@ -1305,19 +1305,19 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -1344,19 +1344,19 @@ define amdgpu_ps <4 x float> @load_mip_cube(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.cube), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -1392,16 +1392,16 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1430,16 +1430,16 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1467,16 +1467,16 @@ define amdgpu_ps <4 x float> @load_mip_1darray(<8 x i32> inreg %rsrc, <2 x i16>
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1512,19 +1512,19 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>)
@@ -1552,19 +1552,19 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -1591,19 +1591,19 @@ define amdgpu_ps <4 x float> @load_mip_2darray(<8 x i32> inreg %rsrc, <2 x i16>
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.mip.2darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -1643,9 +1643,9 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -1671,9 +1671,9 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -1699,9 +1699,9 @@ define amdgpu_ps void @store_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -1733,13 +1733,13 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX9-NEXT: S_ENDPGM 0
@@ -1764,13 +1764,13 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: S_ENDPGM 0
@@ -1795,13 +1795,13 @@ define amdgpu_ps void @store_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
@@ -1834,16 +1834,16 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1872,16 +1872,16 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1909,16 +1909,16 @@ define amdgpu_ps void @store_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2 x
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1954,16 +1954,16 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -1992,16 +1992,16 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2029,16 +2029,16 @@ define amdgpu_ps void @store_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata, <2
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2073,13 +2073,13 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX9-NEXT: S_ENDPGM 0
@@ -2104,13 +2104,13 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: S_ENDPGM 0
@@ -2135,13 +2135,13 @@ define amdgpu_ps void @store_1darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1darray), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
@@ -2174,16 +2174,16 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2212,16 +2212,16 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2249,16 +2249,16 @@ define amdgpu_ps void @store_2darray(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2294,16 +2294,16 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2332,16 +2332,16 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2369,16 +2369,16 @@ define amdgpu_ps void @store_2dmsaa(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2414,19 +2414,19 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>)
@@ -2454,19 +2454,19 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -2493,19 +2493,19 @@ define amdgpu_ps void @store_2darraymsaa(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.2darraymsaa), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -2540,13 +2540,13 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX9-NEXT: S_ENDPGM 0
@@ -2571,13 +2571,13 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: S_ENDPGM 0
@@ -2602,13 +2602,13 @@ define amdgpu_ps void @store_mip_1d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
; GFX12-NEXT: S_ENDPGM 0
@@ -2641,16 +2641,16 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2679,16 +2679,16 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2716,16 +2716,16 @@ define amdgpu_ps void @store_mip_2d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -2761,19 +2761,19 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>)
@@ -2801,19 +2801,19 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -2840,19 +2840,19 @@ define amdgpu_ps void @store_mip_3d(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.3d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -2888,19 +2888,19 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>)
@@ -2928,19 +2928,19 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -2967,19 +2967,19 @@ define amdgpu_ps void @store_mip_cube(<8 x i32> inreg %rsrc, <4 x float> %vdata,
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.cube), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -3015,16 +3015,16 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -3053,16 +3053,16 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -3090,16 +3090,16 @@ define amdgpu_ps void @store_mip_1darray(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF]](i16)
@@ -3135,19 +3135,19 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>)
@@ -3175,19 +3175,19 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -3214,19 +3214,19 @@ define amdgpu_ps void @store_mip_2darray(<8 x i32> inreg %rsrc, <4 x float> %vda
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[COPY13:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr5
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY13]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY13]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR3:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.mip.2darray), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR3]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -3256,9 +3256,9 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3284,9 +3284,9 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3312,9 +3312,9 @@ define amdgpu_ps <4 x float> @getresinfo_1d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3346,9 +3346,9 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3374,9 +3374,9 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3402,9 +3402,9 @@ define amdgpu_ps <4 x float> @getresinfo_2d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3436,9 +3436,9 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.3d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3464,9 +3464,9 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.3d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3492,9 +3492,9 @@ define amdgpu_ps <4 x float> @getresinfo_3d(<8 x i32> inreg %rsrc, <2 x i16> %co
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.3d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3526,9 +3526,9 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.cube), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3554,9 +3554,9 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.cube), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3582,9 +3582,9 @@ define amdgpu_ps <4 x float> @getresinfo_cube(<8 x i32> inreg %rsrc, <2 x i16> %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.cube), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3616,9 +3616,9 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3644,9 +3644,9 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3672,9 +3672,9 @@ define amdgpu_ps <4 x float> @getresinfo_1darray(<8 x i32> inreg %rsrc, <2 x i16
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.1darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3706,9 +3706,9 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3734,9 +3734,9 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3762,9 +3762,9 @@ define amdgpu_ps <4 x float> @getresinfo_2darray(<8 x i32> inreg %rsrc, <2 x i16
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darray), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3796,9 +3796,9 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2dmsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3824,9 +3824,9 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2dmsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3852,9 +3852,9 @@ define amdgpu_ps <4 x float> @getresinfo_2dmsaa(<8 x i32> inreg %rsrc, <2 x i16>
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2dmsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3886,9 +3886,9 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3
@@ -3914,9 +3914,9 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3942,9 +3942,9 @@ define amdgpu_ps <4 x float> @getresinfo_2darraymsaa(<8 x i32> inreg %rsrc, <2 x
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.getresinfo.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1
@@ -3976,9 +3976,9 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable load (f32), addrspace 8)
@@ -4000,9 +4000,9 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (f32), addrspace 8)
@@ -4024,9 +4024,9 @@ define amdgpu_ps float @load_1d_V1(<8 x i32> inreg %rsrc, <2 x i16> %coords) {
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(f32) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 8, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (f32), addrspace 8)
@@ -4054,9 +4054,9 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable load (<2 x f32>), addrspace 8)
@@ -4080,9 +4080,9 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<2 x f32>), addrspace 8)
@@ -4106,9 +4106,9 @@ define amdgpu_ps <2 x float> @load_1d_V2(<8 x i32> inreg %rsrc, <2 x i16> %coord
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<2 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 9, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable load (<2 x f32>), addrspace 8)
@@ -4139,9 +4139,9 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](f32), 2, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable store (f32), addrspace 8)
@@ -4163,9 +4163,9 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](f32), 2, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (f32), addrspace 8)
@@ -4187,9 +4187,9 @@ define amdgpu_ps void @store_1d_V1(<8 x i32> inreg %rsrc, float %vdata, <2 x i16
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(f32) = COPY $vgpr0
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[COPY8]](f32), 2, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (f32), addrspace 8)
@@ -4219,9 +4219,9 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32)
; GFX9-NEXT: [[COPY10:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr2
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY10]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x f32>), 12, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 3 :: (dereferenceable store (<2 x f32>), addrspace 8)
@@ -4245,9 +4245,9 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32)
; GFX10NSA-NEXT: [[COPY10:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr2
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY10]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x f32>), 12, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<2 x f32>), addrspace 8)
@@ -4271,9 +4271,9 @@ define amdgpu_ps void @store_1d_V2(<8 x i32> inreg %rsrc, <2 x float> %vdata, <2
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32)
; GFX12-NEXT: [[COPY10:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr2
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY10]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY10]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<2 x f32>), 12, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 0, 1 :: (dereferenceable store (<2 x f32>), addrspace 8)
@@ -4300,9 +4300,9 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 1, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4328,9 +4328,9 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 1, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4356,9 +4356,9 @@ define amdgpu_ps <4 x float> @load_1d_glc(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 1, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4390,9 +4390,9 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4418,9 +4418,9 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4446,9 +4446,9 @@ define amdgpu_ps <4 x float> @load_1d_slc(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4480,9 +4480,9 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 3, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4508,9 +4508,9 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
; GFX10NSA-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 3, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4536,9 +4536,9 @@ define amdgpu_ps <4 x float> @load_1d_glc_slc(<8 x i32> inreg %rsrc, <2 x i16> %
; GFX12-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<8 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32), [[COPY4]](i32), [[COPY5]](i32), [[COPY6]](i32), [[COPY7]](i32)
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<4 x f32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 3, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4575,9 +4575,9 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 1, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4603,9 +4603,9 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 1, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4631,9 +4631,9 @@ define amdgpu_ps void @store_1d_glc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 1, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4665,9 +4665,9 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4693,9 +4693,9 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4721,9 +4721,9 @@ define amdgpu_ps void @store_1d_slc(<8 x i32> inreg %rsrc, <4 x float> %vdata, <
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 2, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4755,9 +4755,9 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX9-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX9-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 3, 3 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4783,9 +4783,9 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX10NSA-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX10NSA-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 3, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4811,9 +4811,9 @@ define amdgpu_ps void @store_1d_glc_slc(<8 x i32> inreg %rsrc, <4 x float> %vdat
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<4 x f32>) = G_BUILD_VECTOR [[COPY8]](f32), [[COPY9]](f32), [[COPY10]](f32), [[COPY11]](f32)
; GFX12-NEXT: [[COPY12:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr4
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY12]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY12]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF]](i16)
; GFX12-NEXT: G_AMDGPU_INTRIN_IMAGE_STORE intrinsic(@llvm.amdgcn.image.store.1d), [[BUILD_VECTOR1]](<4 x f32>), 15, [[BUILD_VECTOR2]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 0, 3, 1 :: (dereferenceable store (<4 x f32>), addrspace 8)
@@ -4880,9 +4880,9 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF1]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 1, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4914,9 +4914,9 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX10NSA-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF1]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 1, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4948,9 +4948,9 @@ define amdgpu_ps <4 x float> @load_1d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
; GFX12-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[DEF1]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.1d), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR]](<8 x i32>), 1, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -4991,13 +4991,13 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 3 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX9-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32), [[UV4:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x i32>)
@@ -5028,13 +5028,13 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX10NSA-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32), [[UV4:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x i32>)
@@ -5065,13 +5065,13 @@ define amdgpu_ps <4 x float> @load_2d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[COPY8:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr0
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i16>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
; GFX12-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32), [[UV4:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD]](<5 x i32>)
@@ -5113,16 +5113,16 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF1]](i16)
@@ -5157,16 +5157,16 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF1]](i16)
@@ -5200,16 +5200,16 @@ define amdgpu_ps <4 x float> @load_3d_tfe(<8 x i32> inreg %rsrc, <2 x i16> %coor
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[DEF1:%[0-9]+]]:_(i16) = G_IMPLICIT_DEF
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[DEF1]](i16)
@@ -5254,19 +5254,19 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
; GFX9-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX9-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX9-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX9-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX9-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX9-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX9-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX9-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX9-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX9-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x f16>) = G_CONCAT_VECTORS [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>)
@@ -5300,19 +5300,19 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
; GFX10NSA-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX10NSA-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX10NSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX10NSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX10NSA-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX10NSA-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX10NSA-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX10NSA-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX10NSA-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX10NSA-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX10NSA-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
@@ -5345,19 +5345,19 @@ define amdgpu_ps <4 x float> @load_2darraymsaa_tfe(<8 x i32> inreg %rsrc, <2 x i
; GFX12-NEXT: [[COPY9:%[0-9]+]]:_(<2 x i16>) = COPY $vgpr1
; GFX12-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GFX12-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
- ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](s32)
- ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
+ ; GFX12-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY8]](<2 x i16>)
; GFX12-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](s32)
- ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](s32)
- ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(s32) = G_BITCAST [[COPY9]](<2 x i16>)
- ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST3]], [[C1]](i32)
- ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](s32)
+ ; GFX12-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC1:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
+ ; GFX12-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
+ ; GFX12-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR2]](i32)
+ ; GFX12-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY9]](<2 x i16>)
+ ; GFX12-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
+ ; GFX12-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR3]](i32)
; GFX12-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC]](i16), [[TRUNC1]](i16)
; GFX12-NEXT: [[BUILD_VECTOR2:%[0-9]+]]:_(<2 x i16>) = G_BUILD_VECTOR [[TRUNC2]](i16), [[TRUNC3]](i16)
; GFX12-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD intrinsic(@llvm.amdgcn.image.load.2darraymsaa), 15, [[BUILD_VECTOR1]](<2 x i16>), [[BUILD_VECTOR2]](<2 x i16>), $noreg, $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 1 :: (dereferenceable load (<4 x f32>), addrspace 8)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
index 61838b2195379..334b9eeb2d85a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
@@ -236,11 +236,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -268,12 +268,12 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
index 3f858543f4692..3ea85832af248 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
@@ -53,8 +53,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
@@ -70,9 +70,9 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
@@ -97,8 +97,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
@@ -115,9 +115,9 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -141,11 +141,11 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -160,13 +160,13 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -189,12 +189,12 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -208,14 +208,14 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -239,8 +239,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -255,9 +255,9 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -280,9 +280,9 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
@@ -294,10 +294,10 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -321,8 +321,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -337,9 +337,9 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -363,8 +363,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
@@ -380,9 +380,9 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
@@ -407,8 +407,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
@@ -425,9 +425,9 @@ body: |
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -451,11 +451,11 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -470,13 +470,13 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -499,12 +499,12 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -518,14 +518,14 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -548,9 +548,9 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
@@ -564,10 +564,10 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[DEF]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -590,9 +590,9 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
@@ -605,10 +605,10 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[DEF:%[0-9]+]]:_(s16) = G_IMPLICIT_DEF
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[DEF]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
@@ -632,9 +632,9 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
@@ -646,10 +646,10 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -673,10 +673,10 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C1]]
@@ -692,12 +692,12 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -721,8 +721,8 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -737,9 +737,9 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -762,12 +762,12 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -781,14 +781,14 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -812,11 +812,11 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -831,13 +831,13 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -860,12 +860,12 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -879,14 +879,14 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -910,11 +910,11 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -929,13 +929,13 @@ body: |
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -1020,16 +1020,16 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
- ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+ ; GFX8-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST]](<2 x i32>)
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C1]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s32>)
- ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C1]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX8-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x i32>)
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[UV2]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C]](s32)
; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
@@ -1043,18 +1043,18 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST]](<2 x i32>)
+ ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C1]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s32>)
- ; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C1]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x i32>)
+ ; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[UV2]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
@@ -1077,15 +1077,15 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
- ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
- ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+ ; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX8-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST]](<2 x i32>)
+ ; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
- ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
- ; GFX8-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s32>)
- ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
- ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C]](s32)
+ ; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX8-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x i32>)
+ ; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[UV3]](i32)
+ ; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C]](s32)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
@@ -1097,17 +1097,17 @@ body: |
; GFX9-NEXT: {{ $}}
; GFX9-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; GFX9-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr2_vgpr3
- ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](<2 x s32>)
- ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[UV]](s32)
+ ; GFX9-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX9-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST]](<2 x i32>)
+ ; GFX9-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[UV]](i32)
; GFX9-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY2]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](s32)
- ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s32>) = G_BITCAST [[COPY]](<4 x s16>)
- ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s32>)
- ; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
- ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
- ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](s32)
+ ; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
+ ; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i32>) = G_BITCAST [[COPY]](<4 x s16>)
+ ; GFX9-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x i32>)
+ ; GFX9-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[UV3]](i32)
+ ; GFX9-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C]](s32)
+ ; GFX9-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX9-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s16>) = G_BUILD_VECTOR [[TRUNC]](s16), [[TRUNC1]](s16)
; GFX9-NEXT: $vgpr0 = COPY [[BUILD_VECTOR]](<2 x s16>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 394ba5e906f1a..73695793e8faf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
@@ -532,14 +532,15 @@ define <2 x i16> @v_mul_v2i16(<2 x i16> %num, <2 x i16> %den) {
; GFX7-LABEL: v_mul_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_mul_u32_u24_e32 v0, v0, v1
-; GFX7-NEXT: v_mul_u32_u24_e32 v1, v3, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_mul_u32_u24_e32 v1, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 884cf3b9200b5..df17904c3556d 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16,-sramecc < %s | FileCheck -check-prefixes=GFX1250,GFX1250-NOECC,GFX1250-NOECC-SDAG-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16,-sramecc < %s | FileCheck -check-prefixes=GFX1250,GFX1250-NOECC,GFX1250-NOECC-SDAG-FAKE16 %s
@@ -2421,15 +2421,15 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2441,16 +2441,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_reg_hi:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -2479,15 +2469,15 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2499,16 +2489,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -2538,15 +2518,15 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2558,16 +2538,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -2597,15 +2567,15 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2617,16 +2587,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_zexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -2664,7 +2624,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
@@ -2684,7 +2646,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
@@ -2723,7 +2687,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
@@ -2743,7 +2709,9 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
@@ -3024,15 +2992,17 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -3046,18 +3016,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3]
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_reg_hi:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -3086,15 +3044,17 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -3108,18 +3068,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_reg_hi_immneg128(ptr inre
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u16 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -3149,15 +3097,17 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -3171,18 +3121,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3]
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -3212,15 +3150,17 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
-; GFX1250-GISEL-FAKE16: ; %bb.0:
-; GFX1250-GISEL-FAKE16-NEXT: global_wb
-; GFX1250-GISEL-FAKE16-NEXT: v_nop
-; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
-; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: global_wb
+; GFX1250-GISEL-NEXT: v_nop
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -3234,18 +3174,6 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
-; GFX1250-GISEL-TRUE16: ; %bb.0:
-; GFX1250-GISEL-TRUE16-NEXT: global_wb
-; GFX1250-GISEL-TRUE16-NEXT: v_nop
-; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-TRUE16-NEXT: flat_load_u8 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v1
-; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
-;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_zexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: global_wb
@@ -3283,7 +3211,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
@@ -3305,9 +3236,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
@@ -3346,7 +3278,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
@@ -3368,9 +3303,10 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-TRUE16-NEXT: v_mov_b32_e32 v0, v1
+; GFX1250-GISEL-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
index 1f986f8d07786..f8e6ae4ab86bf 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-store.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-FAKE16 %s
; xUN: llc -global-isel=1 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG,GFX1250-SDAG-REAL16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-mesa-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL,GFX1250-GISEL-REAL16 %s
; Test using saddr addressing mode of flat_*store_* instructions.
@@ -1365,13 +1365,14 @@ define amdgpu_ps void @atomic_flat_store_saddr_i64_zext_vgpr_offset_neg128(ptr i
; --------------------------------------------------------------------------------
define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
-; GFX1250-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_wb
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
+; GFX1250-SDAG-NEXT: s_endpgm
+;
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: global_wb
@@ -1379,6 +1380,15 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3]
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-GISEL-REAL16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr:
+; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
+; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v0, v1, s[2:3]
+; GFX1250-GISEL-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%data.hi = extractelement <2 x i16> %data, i32 1
@@ -1387,13 +1397,14 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr(ptr inreg %sbase, i3
}
define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr inreg %sbase, i32 %voffset, <2 x i16> %data) {
-; GFX1250-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: global_wb
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: global_wb
+; GFX1250-SDAG-NEXT: v_nop
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
+; GFX1250-SDAG-NEXT: s_endpgm
+;
; GFX1250-GISEL-FAKE16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
; GFX1250-GISEL-FAKE16: ; %bb.0:
; GFX1250-GISEL-FAKE16-NEXT: global_wb
@@ -1401,6 +1412,15 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128(ptr in
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-GISEL-FAKE16-NEXT: flat_store_d16_hi_b16 v0, v1, s[2:3] offset:-128
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-GISEL-REAL16-LABEL: flat_store_saddr_i16_d16hi_zext_vgpr_offset_neg128:
+; GFX1250-GISEL-REAL16: ; %bb.0:
+; GFX1250-GISEL-REAL16-NEXT: global_wb
+; GFX1250-GISEL-REAL16-NEXT: v_nop
+; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX1250-GISEL-REAL16-NEXT: flat_store_b16 v0, v1, s[2:3] offset:-128
+; GFX1250-GISEL-REAL16-NEXT: s_endpgm
%zext.offset = zext i32 %voffset to i64
%gep0 = getelementptr inbounds i8, ptr %sbase, i64 %zext.offset
%gep1 = getelementptr inbounds i8, ptr %gep0, i64 -128
@@ -1442,6 +1462,5 @@ define amdgpu_ps void @flat_store_saddr_i16_d16hi_trunci8_zext_vgpr_offset_neg12
ret void
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1250-GISEL-REAL16: {{.*}}
; GFX1250-SDAG-FAKE16: {{.*}}
; GFX1250-SDAG-REAL16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
index 6f0d53787f1f9..b3ca80771144c 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.v2f16.no.fast.math.ll
@@ -116,15 +116,25 @@ define half @fptrunc_v3f32_v3f16_extract_uses(<3 x float> %vec_float) {
}
define half @fptrunc_v4f32_v4f16_extract_uses(<4 x float> %vec_float) {
-; GFX950-LABEL: fptrunc_v4f32_v4f16_extract_uses:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cvt_pk_f16_f32 v2, v2, v3
-; GFX950-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: fptrunc_v4f32_v4f16_extract_uses:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v2, v2, v3
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v4f32_v4f16_extract_uses:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%vec_half = fptrunc <4 x float> %vec_float to <4 x half>
%f0 = extractelement <4 x half> %vec_half, i64 0
%f1 = extractelement <4 x half> %vec_half, i64 1
@@ -137,21 +147,37 @@ define half @fptrunc_v4f32_v4f16_extract_uses(<4 x float> %vec_float) {
}
define half @fptrunc_v8f32_v8f16_extract_uses(<8 x float> %vec_float) {
-; GFX950-LABEL: fptrunc_v8f32_v8f16_extract_uses:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cvt_pk_f16_f32 v6, v6, v7
-; GFX950-NEXT: v_cvt_pk_f16_f32 v4, v4, v5
-; GFX950-NEXT: v_cvt_pk_f16_f32 v2, v2, v3
-; GFX950-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_e32 v1, v2, v3
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: fptrunc_v8f32_v8f16_extract_uses:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v6, v6, v7
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v4, v4, v5
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v2, v2, v3
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v1, v2, v3
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v8f32_v8f16_extract_uses:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v2, v4, v5
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v3, v6, v7
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v1, v2, v3
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%vec_half = fptrunc <8 x float> %vec_float to <8 x half>
%f0 = extractelement <8 x half> %vec_half, i64 0
%f1 = extractelement <8 x half> %vec_half, i64 1
@@ -172,33 +198,61 @@ define half @fptrunc_v8f32_v8f16_extract_uses(<8 x float> %vec_float) {
}
define half @fptrunc_v16f32_v16f16_extract_uses(<16 x float> %vec_float) {
-; GFX950-LABEL: fptrunc_v16f32_v16f16_extract_uses:
-; GFX950: ; %bb.0:
-; GFX950-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX950-NEXT: v_cvt_pk_f16_f32 v14, v14, v15
-; GFX950-NEXT: v_cvt_pk_f16_f32 v12, v12, v13
-; GFX950-NEXT: v_cvt_pk_f16_f32 v10, v10, v11
-; GFX950-NEXT: v_cvt_pk_f16_f32 v8, v8, v9
-; GFX950-NEXT: v_cvt_pk_f16_f32 v6, v6, v7
-; GFX950-NEXT: v_cvt_pk_f16_f32 v4, v4, v5
-; GFX950-NEXT: v_cvt_pk_f16_f32 v2, v2, v3
-; GFX950-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v4, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v5, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v6, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_sdwa v7, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_e32 v1, v2, v3
-; GFX950-NEXT: v_add_f16_e32 v2, v4, v5
-; GFX950-NEXT: v_add_f16_e32 v3, v6, v7
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: v_add_f16_e32 v1, v2, v3
-; GFX950-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX950-NEXT: s_setpc_b64 s[30:31]
+; GFX950-SDAG-LABEL: fptrunc_v16f32_v16f16_extract_uses:
+; GFX950-SDAG: ; %bb.0:
+; GFX950-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v14, v14, v15
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v12, v12, v13
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v10, v10, v11
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v8, v8, v9
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v6, v6, v7
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v4, v4, v5
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v2, v2, v3
+; GFX950-SDAG-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v1, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v2, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v3, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v4, v8, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v5, v10, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v6, v12, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_sdwa v7, v14, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v1, v2, v3
+; GFX950-SDAG-NEXT: v_add_f16_e32 v2, v4, v5
+; GFX950-SDAG-NEXT: v_add_f16_e32 v3, v6, v7
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: v_add_f16_e32 v1, v2, v3
+; GFX950-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX950-GISEL-LABEL: fptrunc_v16f32_v16f16_extract_uses:
+; GFX950-GISEL: ; %bb.0:
+; GFX950-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v1, v2, v3
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v2, v4, v5
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v3, v6, v7
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v4, v8, v9
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v5, v10, v11
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v6, v12, v13
+; GFX950-GISEL-NEXT: v_cvt_pk_f16_f32 v7, v14, v15
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v1, v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v2, v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v4, v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v5, v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v6, v6, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_sdwa v7, v7, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v1, v2, v3
+; GFX950-GISEL-NEXT: v_add_f16_e32 v2, v4, v5
+; GFX950-GISEL-NEXT: v_add_f16_e32 v3, v6, v7
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: v_add_f16_e32 v1, v2, v3
+; GFX950-GISEL-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX950-GISEL-NEXT: s_setpc_b64 s[30:31]
%vec_half = fptrunc <16 x float> %vec_float to <16 x half>
%f0 = extractelement <16 x half> %vec_half, i64 0
%f1 = extractelement <16 x half> %vec_half, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.ll b/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.ll
index 0748480c3ff4a..557d1adba5b95 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-reg-read-imm-idx.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgpu9.0a -O1 -global-isel -global-isel-abort=2 < %s | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.0a -O1 -global-isel < %s | FileCheck %s
; Test that V_INDIRECT_REG_READ_GPR_IDX expansion handles immediate index operands.
; The wave.reduce.umin with constant arguments folds to 0, which becomes an
@@ -9,10 +9,40 @@
define amdgpu_kernel void @indirect_reg_read_imm_idx() {
; CHECK-LABEL: indirect_reg_read_imm_idx:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: v_mov_b32_e32 v0, 0
-; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, 0
-; CHECK-NEXT: flat_store_short v[0:1], v2
+; CHECK-NEXT: v_pk_mov_b32 v[0:1], 0, 0
+; CHECK-NEXT: v_mov_b32_e32 v4, 16
+; CHECK-NEXT: v_mov_b32_e32 v5, 0
+; CHECK-NEXT: v_mov_b32_e32 v8, 32
+; CHECK-NEXT: v_mov_b32_e32 v9, 0
+; CHECK-NEXT: v_mov_b32_e32 v12, 48
+; CHECK-NEXT: v_mov_b32_e32 v13, 0
+; CHECK-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: flat_load_dwordx4 v[8:11], v[8:9]
+; CHECK-NEXT: s_nop 0
+; CHECK-NEXT: flat_load_dwordx4 v[12:15], v[12:13]
+; CHECK-NEXT: s_mov_b32 s0, 0xffff
+; CHECK-NEXT: v_pk_mov_b32 v[16:17], 0, 0
+; CHECK-NEXT: v_mov_b32_e32 v18, 16
+; CHECK-NEXT: v_mov_b32_e32 v19, 0
+; CHECK-NEXT: v_mov_b32_e32 v20, 32
+; CHECK-NEXT: v_mov_b32_e32 v21, 0
+; CHECK-NEXT: v_mov_b32_e32 v22, 48
+; CHECK-NEXT: v_mov_b32_e32 v23, 0
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; CHECK-NEXT: s_set_gpr_idx_on 0, gpr_idx(SRC0)
+; CHECK-NEXT: v_mov_b32_e32 v24, v0
+; CHECK-NEXT: s_set_gpr_idx_off
+; CHECK-NEXT: v_bfi_b32 v24, s0, 0, v24
+; CHECK-NEXT: s_set_gpr_idx_on 0, gpr_idx(DST)
+; CHECK-NEXT: v_mov_b32_e32 v0, v24
+; CHECK-NEXT: s_set_gpr_idx_off
+; CHECK-NEXT: flat_store_dwordx4 v[16:17], v[0:3]
+; CHECK-NEXT: flat_store_dwordx4 v[18:19], v[4:7]
+; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11]
+; CHECK-NEXT: flat_store_dwordx4 v[22:23], v[12:15]
; CHECK-NEXT: s_endpgm
entry:
%vec = load <32 x i16>, ptr null, align 64
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 643a95385f4e4..8c89db87ebd78 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX6,GFX67-SDAG,GFX6-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX6,GFX67-GISEL,GFX6-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX6,GFX67-GISEL,GFX6-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX7,GFX67-SDAG,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX7,GFX67-GISEL,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX7,GFX67-GISEL,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
@@ -621,42 +621,96 @@ entry:
}
define <2 x i16> @clpeak_imad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) {
-; GFX67-LABEL: clpeak_imad_pat_v2i16:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v3
-; GFX67-NEXT: v_mad_u32_u24 v0, v4, v1, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v5, v2, v3
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v6, v0, v1
-; GFX67-NEXT: v_mad_u32_u24 v4, v4, v1, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v3, v2
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_mad_u32_u24 v5, v5, v2, 1
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, 1
-; GFX67-NEXT: v_mad_u32_u24 v1, v3, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v5
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v4
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v7
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v4, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v3, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v2, v1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_imad_pat_v2i16:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v4, v1, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v5, v2, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v4, v1, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v3, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v5, v5, v2, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v3, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v4, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v2, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: clpeak_imad_pat_v2i16:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v3, v1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v4, v5, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v3, v1, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v2, v5, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v7, v2, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v2, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: clpeak_imad_pat_v2i16:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -1235,89 +1289,192 @@ entry:
}
define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; GFX67-LABEL: clpeak_imad_pat_v4i16:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX67-NEXT: v_add_i32_e32 v7, vcc, 1, v7
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX67-NEXT: v_and_b32_e32 v11, 0xffff, v7
-; GFX67-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mad_u32_u24 v7, v11, v4, v7
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_and_b32_e32 v9, 0xffff, v1
-; GFX67-NEXT: v_add_i32_e32 v6, vcc, 1, v6
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX67-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v10, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v13, v11, v4
-; GFX67-NEXT: v_mul_u32_u24_e32 v4, v7, v4
-; GFX67-NEXT: v_mad_u32_u24 v7, v9, v3, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v12, v10, v5
-; GFX67-NEXT: v_mad_u32_u24 v1, v9, v3, v1
-; GFX67-NEXT: v_mad_u32_u24 v0, v8, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v6, v10, v5, v6
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX67-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; GFX67-NEXT: v_mad_u32_u24 v8, v8, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX67-NEXT: v_or_b32_e32 v7, v9, v7
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX67-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v10, v0, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v5, v6, v5
-; GFX67-NEXT: s_mov_b32 s4, 0x10000
-; GFX67-NEXT: v_or_b32_e32 v8, v9, v8
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v2, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v6, v1, v3
-; GFX67-NEXT: v_add_i32_e32 v8, vcc, s4, v8
-; GFX67-NEXT: v_mad_u32_u24 v1, v1, v3, 1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX67-NEXT: v_add_i32_e32 v7, vcc, s4, v7
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX67-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX67-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX67-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v3, 16, v7
-; GFX67-NEXT: v_mul_u32_u24_e32 v8, v9, v8
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v5, v2
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v7
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, s4, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v5, v5, v6
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v4, v3
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v8
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v7, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v2, v4
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v4, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v6
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_imad_pat_v4i16:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v7, vcc, 1, v7
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v11, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v7, v11, v4, v7
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v9, 0xffff, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v6, vcc, 1, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v10, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v13, v11, v4
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v4, v7, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v7, v9, v3, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v12, v10, v5
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v9, v3, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v8, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v6, v10, v5, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v8, v8, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX67-SDAG-NEXT: v_or_b32_e32 v7, v9, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v10, v0, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v5, v6, v5
+; GFX67-SDAG-NEXT: s_mov_b32 s4, 0x10000
+; GFX67-SDAG-NEXT: v_or_b32_e32 v8, v9, v8
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v6, v1, v3
+; GFX67-SDAG-NEXT: v_add_i32_e32 v8, vcc, s4, v8
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX67-SDAG-NEXT: v_add_i32_e32 v7, vcc, s4, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v9, 0xffff, v10
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-SDAG-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v8, v9, v8
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v5, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_add_i32_e32 v1, vcc, s4, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v5, v5, v6
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v4, v3
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v7, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v2, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v4, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v6
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: clpeak_imad_pat_v4i16:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v4
+; GFX67-GISEL-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v9, v9, v10
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v10, v6, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v9
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v8, v2, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v5, v11, v7, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v9, v3, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v12, v0, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v8, v8, v2, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v10, v10, v6, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v6, 1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v14, v1, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v9, v9, v3, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v11, v11, v7, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v5, v7, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v11
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v13, v4, v6
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v15, v5, v7
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v12
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v8
+; GFX67-GISEL-NEXT: v_or_b32_e32 v9, v9, v10
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v13
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v5, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v14
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v9
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v5, v5, v6
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v15
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v6, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v6
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v4, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v3, v7
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v4, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: clpeak_imad_pat_v4i16:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -1988,42 +2145,96 @@ entry:
}
define <2 x i16> @clpeak_umad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) {
-; GFX67-LABEL: clpeak_umad_pat_v2i16:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v3
-; GFX67-NEXT: v_mad_u32_u24 v0, v4, v1, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v5, v2, v3
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v6, v0, v1
-; GFX67-NEXT: v_mad_u32_u24 v4, v4, v1, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v3, v2
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_mad_u32_u24 v5, v5, v2, 1
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, 1
-; GFX67-NEXT: v_mad_u32_u24 v1, v3, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v5
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v4
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v7
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v4, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v3, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v2, v1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_umad_pat_v2i16:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v4, v1, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v5, v2, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v4, v1, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v3, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v5, v5, v2, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v3, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v4, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v2, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: clpeak_umad_pat_v2i16:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v3, v1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v4, v5, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v3, v1, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v2, v5, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v7, v2, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v2, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: clpeak_umad_pat_v2i16:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -2602,89 +2813,192 @@ entry:
}
define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; GFX67-LABEL: clpeak_umad_pat_v4i16:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; GFX67-NEXT: v_add_i32_e32 v7, vcc, 1, v7
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX67-NEXT: v_and_b32_e32 v11, 0xffff, v7
-; GFX67-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mad_u32_u24 v7, v11, v4, v7
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_and_b32_e32 v9, 0xffff, v1
-; GFX67-NEXT: v_add_i32_e32 v6, vcc, 1, v6
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX67-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v10, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v13, v11, v4
-; GFX67-NEXT: v_mul_u32_u24_e32 v4, v7, v4
-; GFX67-NEXT: v_mad_u32_u24 v7, v9, v3, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v12, v10, v5
-; GFX67-NEXT: v_mad_u32_u24 v1, v9, v3, v1
-; GFX67-NEXT: v_mad_u32_u24 v0, v8, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v6, v10, v5, v6
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX67-NEXT: v_lshlrev_b32_e32 v9, 16, v13
-; GFX67-NEXT: v_mad_u32_u24 v8, v8, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX67-NEXT: v_or_b32_e32 v7, v9, v7
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX67-NEXT: v_lshlrev_b32_e32 v9, 16, v12
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v10, v0, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v5, v6, v5
-; GFX67-NEXT: s_mov_b32 s4, 0x10000
-; GFX67-NEXT: v_or_b32_e32 v8, v9, v8
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v2, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v6, v1, v3
-; GFX67-NEXT: v_add_i32_e32 v8, vcc, s4, v8
-; GFX67-NEXT: v_mad_u32_u24 v1, v1, v3, 1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX67-NEXT: v_add_i32_e32 v7, vcc, s4, v7
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX67-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; GFX67-NEXT: v_and_b32_e32 v9, 0xffff, v10
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX67-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v3, 16, v7
-; GFX67-NEXT: v_mul_u32_u24_e32 v8, v9, v8
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v5, v2
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v7
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, s4, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v5, v5, v6
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v4, v3
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v8
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v7, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v2, v4
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v4, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v6
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_umad_pat_v4i16:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v7, vcc, 1, v7
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v11, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v7, v11, v4, v7
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v9, 0xffff, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v6, vcc, 1, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v10, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v13, v11, v4
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v4, v7, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v7, v9, v3, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v12, v10, v5
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v9, v3, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v8, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v6, v10, v5, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v8, v8, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX67-SDAG-NEXT: v_or_b32_e32 v7, v9, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v12
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v10, v0, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v5, v6, v5
+; GFX67-SDAG-NEXT: s_mov_b32 s4, 0x10000
+; GFX67-SDAG-NEXT: v_or_b32_e32 v8, v9, v8
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v6, v1, v3
+; GFX67-SDAG-NEXT: v_add_i32_e32 v8, vcc, s4, v8
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX67-SDAG-NEXT: v_add_i32_e32 v7, vcc, s4, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v9, 0xffff, v10
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-SDAG-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v8, v9, v8
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v5, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_add_i32_e32 v1, vcc, s4, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v5, v5, v6
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v4, v3
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v7, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v2, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v4, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v6
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: clpeak_umad_pat_v4i16:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v5, vcc, 1, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v4
+; GFX67-GISEL-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v8, v8, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v9, v9, v10
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v8
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v10, v6, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v9
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v8, v2, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v5, v11, v7, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v9, v3, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v12, v0, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v8, v8, v2, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v10, v10, v6, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v6, 1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v14, v1, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v9, v9, v3, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v11, v11, v7, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v5, v7, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v8, v8, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v11
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v13, v4, v6
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v15, v5, v7
+; GFX67-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v12
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v8
+; GFX67-GISEL-NEXT: v_or_b32_e32 v9, v9, v10
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v13
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v5, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v14
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v9
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v9
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v5, v5, v6
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v15
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v6, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v6
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v4, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v3, v7
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v4, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: clpeak_umad_pat_v4i16:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -10509,66 +10823,160 @@ entry:
}
define <2 x i16> @clpeak_imad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) {
-; GFX67-LABEL: clpeak_imad_pat_v2i16_x2:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v3
-; GFX67-NEXT: v_mad_u32_u24 v0, v4, v1, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v5, v2, v3
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1
-; GFX67-NEXT: v_mad_u32_u24 v1, v4, v1, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v2
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX67-NEXT: v_mad_u32_u24 v2, v5, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v2
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mad_u32_u24 v1, v0, v4, v1
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mad_u32_u24 v2, v3, v5, v2
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v1, v0
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v4, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v2, v3
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v3, v5, 1
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mad_u32_u24 v0, v1, v4, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v2, v5, v3
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v6, v0, v1
-; GFX67-NEXT: v_mad_u32_u24 v4, v1, v4, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v3, v2
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_mad_u32_u24 v5, v2, v5, 1
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, 1
-; GFX67-NEXT: v_mad_u32_u24 v1, v3, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v5
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v4
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v7
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v4, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v3, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v2, v1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_imad_pat_v2i16_x2:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v4, v1, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v5, v2, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v4, v1, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v5, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v0, v4, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v3, v5, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v1, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v4, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v3, v5, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v1, v4, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v2, v5, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v1, v4, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v3, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v5, v2, v5, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v3, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v4, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v2, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: clpeak_imad_pat_v2i16_x2:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v3, v1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v3, v1, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v4, v5, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v2, v5, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v0, v4, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v3, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v2, v5, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v1, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v4, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v3, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v1, v4, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v5, v3, v5, 1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v1, v4, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v2, v3, 1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v7, v2, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v2, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: clpeak_imad_pat_v2i16_x2:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -10764,66 +11172,160 @@ entry:
}
define <2 x i16> @clpeak_umad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) {
-; GFX67-LABEL: clpeak_umad_pat_v2i16_x2:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX67-NEXT: v_add_i32_e32 v3, vcc, 1, v3
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v3
-; GFX67-NEXT: v_mad_u32_u24 v0, v4, v1, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v5, v2, v3
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1
-; GFX67-NEXT: v_mad_u32_u24 v1, v4, v1, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v2
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX67-NEXT: v_mad_u32_u24 v2, v5, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v2
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mad_u32_u24 v1, v0, v4, v1
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mad_u32_u24 v2, v3, v5, v2
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v1, v0
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v4, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v2, v3
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v3, v5, 1
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mad_u32_u24 v0, v1, v4, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mad_u32_u24 v3, v2, v5, v3
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_mul_u32_u24_e32 v6, v0, v1
-; GFX67-NEXT: v_mad_u32_u24 v4, v1, v4, 1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v3, v2
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_mad_u32_u24 v5, v2, v5, 1
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, 1
-; GFX67-NEXT: v_mad_u32_u24 v1, v3, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v5
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v4
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v7
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v4, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v3, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v2, v1
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_umad_pat_v2i16_x2:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v3, vcc, 1, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v4, v1, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v5, v2, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v4, v1, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v5, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v0, v4, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v3, v5, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v1, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v4, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v3, v5, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v1, v4, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v3, v2, v5, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v1, v4, 1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v3, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v5, v2, v5, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v3, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v5
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v4, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v2, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: clpeak_umad_pat_v2i16_x2:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v2, vcc, 1, v2
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v3, v1, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v3, v1, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v4, v5, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v3, v2, v5, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v0, v4, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v3, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v2, v5, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v1, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v4, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX67-GISEL-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v3, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v1, v4, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v5, v3, v5, 1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v1, v4, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v1, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v2, v3, 1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v7, v2, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v6
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v2, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v1, v3
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v2, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: clpeak_umad_pat_v2i16_x2:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -11554,26 +12056,49 @@ entry:
}
define <4 x i16> @multi_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i16> %z0, <2 x i16> %z1) {
-; GFX67-LABEL: multi_use_mul_mad_v2i16_var:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX67-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX67-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mad_u32_u24 v0, v7, v6, v5
-; GFX67-NEXT: v_mad_u32_u24 v2, v8, v1, v2
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX67-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v2, v7, v6, v4
-; GFX67-NEXT: v_mad_u32_u24 v1, v8, v1, v3
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: multi_use_mul_mad_v2i16_var:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v7, v6, v5
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v8, v1, v2
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v7, v6, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v8, v1, v3
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: multi_use_mul_mad_v2i16_var:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v4, v5, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v6, v1, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v6, v1, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: multi_use_mul_mad_v2i16_var:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -11667,28 +12192,53 @@ entry:
}
define <2 x i16> @other_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i16> %z, ptr addrspace(3) %ptr) {
-; GFX67-LABEL: other_use_mul_mad_v2i16_var:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX67-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX67-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v0, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v8, v6, v5
-; GFX67-NEXT: v_mad_u32_u24 v4, v6, v5, v4
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, v2
-; GFX67-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX67-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX67-NEXT: s_mov_b32 m0, -1
-; GFX67-NEXT: ds_write_b32 v3, v7
-; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: other_use_mul_mad_v2i16_var:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v0, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v8, v6, v5
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v6, v5, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v1, v2
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_or_b32_e32 v7, v7, v8
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX67-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX67-SDAG-NEXT: ds_write_b32 v3, v7
+; GFX67-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX67-GISEL-LABEL: other_use_mul_mad_v2i16_var:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v7, v4, v5
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v6, v0, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX67-GISEL-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v1, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v4, v5, v7
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX67-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX67-GISEL-NEXT: ds_write_b32 v3, v6
+; GFX67-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: other_use_mul_mad_v2i16_var:
; GFX8-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
index 21432526d0e9b..b688bab927630 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.f16.fp8.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG-REAL16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-REAL16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 %s -o - | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-FAKE16 %s
declare half @llvm.amdgcn.cvt.f16.bf8(i32, i32)
declare half @llvm.amdgcn.cvt.f16.fp8(i32, i32)
@@ -681,7 +681,9 @@ define amdgpu_ps float @test_cvt_pk_f16_fp8_v_hi(<2 x i16> %a) {
; GFX1250-GISEL-REAL16-NEXT: global_wb
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_f16_fp8 v0, v0.h
+; GFX1250-GISEL-REAL16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_pk_f16_fp8 v0, v0.l
; GFX1250-GISEL-REAL16-NEXT: ; return to shader part epilog
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_pk_f16_fp8_v_hi:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
index 96d12677cdc88..d6560da616246 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.fp8.f16.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG-REAL16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-REAL16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-GISEL-FAKE16 %s
declare i16 @llvm.amdgcn.cvt.pk.bf8.f16(<2 x half>)
declare i16 @llvm.amdgcn.cvt.pk.fp8.f16(<2 x half>)
@@ -487,9 +487,11 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
; GFX1250-GISEL-REAL16-NEXT: global_wb
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
-; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.h, v1
-; GFX1250-GISEL-REAL16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-REAL16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v7, v4
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_bf8_f16 v2, v0.l, v1
+; GFX1250-GISEL-REAL16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-REAL16-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_bf8_f16_hi_byte0:
@@ -497,11 +499,11 @@ define amdgpu_ps void @test_cvt_sr_bf8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_lshrrev_b32 v0, 16, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_bf8_f16 v2, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%a.1 = extractelement <2 x half> %a, i32 1
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.bf8.f16(half %a.1, i32 %sr, i32 %old, i32 0)
@@ -733,9 +735,11 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
; GFX1250-GISEL-REAL16-NEXT: global_wb
; GFX1250-GISEL-REAL16-NEXT: v_nop
; GFX1250-GISEL-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-REAL16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v4, v3
-; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.h, v1
-; GFX1250-GISEL-REAL16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-REAL16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
+; GFX1250-GISEL-REAL16-NEXT: v_mov_b32_e32 v7, v4
+; GFX1250-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-GISEL-REAL16-NEXT: v_cvt_sr_fp8_f16 v2, v0.l, v1
+; GFX1250-GISEL-REAL16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-REAL16-NEXT: s_endpgm
;
; GFX1250-GISEL-FAKE16-LABEL: test_cvt_sr_fp8_f16_hi_byte0:
@@ -743,11 +747,11 @@ define amdgpu_ps void @test_cvt_sr_fp8_f16_hi_byte0(<2 x half> %a, i32 %sr, i32
; GFX1250-GISEL-FAKE16-NEXT: global_wb
; GFX1250-GISEL-FAKE16-NEXT: v_nop
; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-FAKE16-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_lshrrev_b32 v0, 16, v0
-; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v4, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_dual_lshrrev_b32 v0, 16, v0 :: v_dual_mov_b32 v6, v3
+; GFX1250-GISEL-FAKE16-NEXT: v_mov_b32_e32 v7, v4
; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-GISEL-FAKE16-NEXT: v_cvt_sr_fp8_f16 v2, v0, v1
-; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[4:5], v2, off
+; GFX1250-GISEL-FAKE16-NEXT: global_store_b32 v[6:7], v2, off
; GFX1250-GISEL-FAKE16-NEXT: s_endpgm
%a.1 = extractelement <2 x half> %a, i32 1
%cvt = tail call i32 @llvm.amdgcn.cvt.sr.fp8.f16(half %a.1, i32 %sr, i32 %old, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
index eb3ac7a2c53fd..c8c98058bc37a 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-flat.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
-; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(0) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
@@ -65,21 +65,41 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(0) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: flat_load_b32 v0, v[0:1] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -106,21 +126,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(0) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: flat_load_b32 v0, v[0:1] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -150,27 +190,55 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(0) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -205,25 +273,49 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(0) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: flat_load_b64 v[0:1], v[0:1] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b64 v[0:1], v[0:1] scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(0) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -349,24 +441,46 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_min(ptr addrspac
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
-; GFX11-NEXT: flat_load_b32 v0, v[0:1] glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfffff000, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, -1, v1, vcc_lo
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_min:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:-4096 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 -4096
%a = load atomic <2 x i16>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
%e0 = extractelement <2 x i16> %a, i32 0
@@ -396,21 +510,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:4095 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: flat_load_b32 v0, v[0:1] offset:4095 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%gep = getelementptr inbounds i8, ptr addrspace(0) %p, i64 4095
%a = load atomic <2 x i16>, ptr addrspace(0) %gep syncscope("agent") monotonic, align 8
%e0 = extractelement <2 x i16> %a, i32 0
@@ -423,9 +557,5 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
; GFX10-SDAG: {{.*}}
-; GFX11-GISEL: {{.*}}
-; GFX11-SDAG: {{.*}}
-; GFX12-GISEL: {{.*}}
-; GFX12-SDAG: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
index 7b17bb8c760ca..11653be602779 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-global.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
-; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(1) %p, ptr addrspace(1) %out) {
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
@@ -151,21 +151,41 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(1) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -194,24 +214,47 @@ define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(1) %p, ptr
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -237,21 +280,41 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(1) %p
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -278,21 +341,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(1) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -322,24 +405,47 @@ define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(1) %p, ptr
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -366,21 +472,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(1) %p
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -410,27 +536,55 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(1) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -467,30 +621,61 @@ define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(1) %p, ptr
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -524,27 +709,55 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(1) %p
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v1.l, v1.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v1.l, v5.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -579,25 +792,49 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(1) %p, pt
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -635,28 +872,55 @@ define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(1) %p, ptr
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: buffer_gl1_inv
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl1_inv
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_DEV
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl1_inv
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -691,25 +955,49 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(1) %p
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b64 v[0:1], v[0:1], off
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v1.l, v1.h, v0.l
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b64 v[0:1], v[0:1], off
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v4.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v1.l, v5.l, v0.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(1) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -834,21 +1122,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(
; GFX10-NEXT: global_store_short v[2:3], v0, off
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:1 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:1 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 1
%a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
%e0 = extractelement <2 x i16> %a, i32 0
@@ -867,32 +1175,63 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
; GFX9-NEXT: global_store_short v[2:3], v0, off
; GFX9-NEXT: s_endpgm
;
-; GFX10-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX10: ; %bb.0:
-; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
-; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
-; GFX10-NEXT: global_load_dword v0, v[0:1], off offset:2047 glc dlc
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: global_store_short v[2:3], v0, off
-; GFX10-NEXT: s_endpgm
+; GFX10-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10-SDAG: ; %bb.0:
+; GFX10-SDAG-NEXT: v_add_co_u32 v0, vcc_lo, 0x800, v0
+; GFX10-SDAG-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-SDAG-NEXT: global_load_dword v0, v[0:1], off offset:2047 glc dlc
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-SDAG-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: global_store_short v[2:3], v0, off
+; GFX10-SDAG-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: global_store_b16 v[2:3], v0, off
-; GFX11-NEXT: s_endpgm
+; GFX10-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX10-GISEL: ; %bb.0:
+; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, 0xfff, v0
+; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-GISEL-NEXT: global_load_dword v0, v[0:1], off glc dlc
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX10-GISEL-NEXT: v_add_nc_u16 v0, v0, v1
+; GFX10-GISEL-NEXT: global_store_short v[2:3], v0, off
+; GFX10-GISEL-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: global_store_b16 v[2:3], v0, off
-; GFX12-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
+; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:4095 glc
+; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX11-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: global_load_b32 v0, v[0:1], off offset:4095 scope:SCOPE_DEV
+; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
+; GFX12-GISEL-NEXT: global_store_b16 v[2:3], v0, off
+; GFX12-GISEL-NEXT: s_endpgm
%gep = getelementptr inbounds i8, ptr addrspace(1) %p, i64 4095
%a = load atomic <2 x i16>, ptr addrspace(1) %gep syncscope("agent") monotonic, align 8
%e0 = extractelement <2 x i16> %a, i32 0
@@ -903,9 +1242,5 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GCN: {{.*}}
-; GFX11-GISEL: {{.*}}
-; GFX11-SDAG: {{.*}}
-; GFX12-GISEL: {{.*}}
-; GFX12-SDAG: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
index da284e2979a49..bcdafce404563 100644
--- a/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-atomic-local.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-SDAG %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu10.10 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
-; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
+; RUN: llc -mtriple=amdgpu10.10 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG %s
-; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
+; RUN: llc -mtriple=amdgpu11.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL %s
; RUN: llc -mtriple=amdgpu12.00 -global-isel=0 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG %s
-; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
+; RUN: llc -mtriple=amdgpu12.00 -global-isel=1 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL %s
define amdgpu_cs void @atomic_load_f32x2_monotonic_agent(ptr addrspace(3) %p, ptr addrspace(3) %out) {
; GFX9-LABEL: atomic_load_f32x2_monotonic_agent:
@@ -148,21 +148,41 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_agent(ptr addrspace(3) %p, pt
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -189,23 +209,45 @@ define amdgpu_cs void @atomic_load_f16x2_seq_cst_agent(ptr addrspace(3) %p, ptr
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -231,21 +273,41 @@ define amdgpu_cs void @atomic_load_f16x2_monotonic_wavefront(ptr addrspace(3) %p
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x2_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x2_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <2 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
%num1 = extractelement <2 x half> %a0, i32 0
%num2 = extractelement <2 x half> %a0, i32 1
@@ -272,21 +334,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent(ptr addrspace(3) %p, pt
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -314,23 +396,45 @@ define amdgpu_cs void @atomic_load_i16x2_seq_cst_agent(ptr addrspace(3) %p, ptr
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -357,21 +461,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_wavefront(ptr addrspace(3) %p
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a = load atomic <2 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 4
%e0 = extractelement <2 x i16> %a, i32 0
%e1 = extractelement <2 x i16> %a, i32 1
@@ -401,27 +525,55 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_agent(ptr addrspace(3) %p, pt
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b64 v[2:3], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b64 v[2:3], v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -456,29 +608,59 @@ define amdgpu_cs void @atomic_load_f16x4_seq_cst_agent(ptr addrspace(3) %p, ptr
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b64 v[2:3], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b64 v[2:3], v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -512,27 +694,55 @@ define amdgpu_cs void @atomic_load_f16x4_monotonic_wavefront(ptr addrspace(3) %p
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_f16x4_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b64 v[2:3], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
-; GFX11-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_f16x4_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b64 v[2:3], v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
-; GFX12-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: v_mul_f16_e32 v0.h, v3.l, v3.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_f16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mul_f16_e32 v0.h, v3.l, v4.l
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_f16_e32 v0.l, v0.l, v0.h
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x half>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <4 x half> %a0, i32 0
%num2 = extractelement <4 x half> %a0, i32 1
@@ -567,25 +777,49 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_agent(ptr addrspace(3) %p, pt
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_monotonic_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b64 v[2:3], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_monotonic_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b64 v[2:3], v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -621,27 +855,53 @@ define amdgpu_cs void @atomic_load_i16x4_seq_cst_agent(ptr addrspace(3) %p, ptr
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_seq_cst_agent:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b64 v[2:3], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: buffer_gl0_inv
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_seq_cst_agent:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b64 v[2:3], v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: global_inv scope:SCOPE_SE
-; GFX12-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: buffer_gl0_inv
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: global_inv scope:SCOPE_SE
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_seq_cst_agent:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: global_inv scope:SCOPE_SE
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("agent") seq_cst, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -676,25 +936,49 @@ define amdgpu_cs void @atomic_load_i16x4_monotonic_wavefront(ptr addrspace(3) %p
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x4_monotonic_wavefront:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b64 v[2:3], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x4_monotonic_wavefront:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b64 v[2:3], v0
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX11-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v2.l, v2.h
+; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-SDAG-NEXT: v_mad_u16 v0.l, v3.l, v3.h, v0.l
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x4_monotonic_wavefront:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b64 v[2:3], v0
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v2.l, v0.l
+; GFX12-GISEL-NEXT: v_mad_u16 v0.l, v3.l, v4.l, v0.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%a0 = load atomic <4 x i16>, ptr addrspace(3) %p syncscope("wavefront") monotonic, align 8
%num1 = extractelement <4 x i16> %a0, i32 0
%num2 = extractelement <4 x i16> %a0, i32 1
@@ -807,21 +1091,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_1(ptr addrspace(
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0 offset:1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0 offset:1
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_1:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0 offset:1
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 1
%a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
%e0 = extractelement <2 x i16> %a, i32 0
@@ -849,21 +1153,41 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
; GFX10-NEXT: ds_write_b16 v1, v0
; GFX10-NEXT: s_endpgm
;
-; GFX11-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: ds_load_b32 v0, v0 offset:4095
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX11-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX12-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: ds_load_b32 v0, v0 offset:4095
-; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
-; GFX12-NEXT: ds_store_b16 v1, v0
-; GFX12-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX11-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX12-SDAG-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX12-SDAG-NEXT: s_wait_dscnt 0x0
+; GFX12-SDAG-NEXT: v_add_nc_u16 v0.l, v0.l, v0.h
+; GFX12-SDAG-NEXT: ds_store_b16 v1, v0
+; GFX12-SDAG-NEXT: s_endpgm
+;
+; GFX12-GISEL-LABEL: atomic_load_i16x2_monotonic_agent_offset_max:
+; GFX12-GISEL: ; %bb.0:
+; GFX12-GISEL-NEXT: ds_load_b32 v0, v0 offset:4095
+; GFX12-GISEL-NEXT: s_wait_dscnt 0x0
+; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_add_nc_u16 v0.l, v0.l, v2.l
+; GFX12-GISEL-NEXT: ds_store_b16 v1, v0
+; GFX12-GISEL-NEXT: s_endpgm
%gep = getelementptr inbounds i8, ptr addrspace(3) %p, i64 4095
%a = load atomic <2 x i16>, ptr addrspace(3) %gep syncscope("agent") monotonic, align 8
%e0 = extractelement <2 x i16> %a, i32 0
@@ -876,9 +1200,5 @@ define amdgpu_cs void @atomic_load_i16x2_monotonic_agent_offset_max(ptr addrspac
; GCN: {{.*}}
; GFX10-GISEL: {{.*}}
; GFX10-SDAG: {{.*}}
-; GFX11-GISEL: {{.*}}
-; GFX11-SDAG: {{.*}}
-; GFX12-GISEL: {{.*}}
-; GFX12-SDAG: {{.*}}
; GFX9-GISEL: {{.*}}
; GFX9-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index e7ee371aa5d30..88f482a0ac889 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -8,11 +8,11 @@
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1100,GISEL-GFX1100,GISEL-GFX1100-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1100,GISEL-GFX1100,GISEL-GFX1100-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX900,GISEL-GFX900 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.06 < %s | FileCheck -check-prefixes=GFX906,GISEL-GFX906 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GISEL-CI %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1100,GISEL-GFX1100,GISEL-GFX1100-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX900,GISEL-GFX900 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.06 < %s | FileCheck -check-prefixes=GFX906,GISEL-GFX906 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GISEL-CI %s
define half @mixlo_simple(float %src0, float %src1, float %src2) #0 {
; GFX1100-LABEL: mixlo_simple:
@@ -1606,37 +1606,37 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x half> %src0, <2 x half>
; SDAG-GFX1100-FAKE16-NEXT: v_mov_b32_e32 v0, v3
; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX900-NEXT: v_mov_b32_e32 v0, v3
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX900-NEXT: v_mov_b32_e32 v0, v3
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GFX906-NEXT: v_mov_b32_e32 v0, v3
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
+; SDAG-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v3
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v5, v3, v4
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e64 v1, v2 clamp
-; VI-NEXT: v_or_b32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e64 v1, v2 clamp
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
; SDAG-CI: ; %bb.0:
@@ -1667,36 +1667,88 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_lo(<2 x half> %src0, <2 x half>
; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
; GISEL-GFX1100: ; %bb.0:
; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1] clamp
-; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX1100-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
-; GISEL-GFX1100-NEXT: v_mov_b32_e32 v0, v3
+; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GISEL-GFX1100-NEXT: v_mov_b32_e32 v4, v3
+; GISEL-GFX1100-NEXT: v_max_f16_e64 v3, v3, v3 clamp
+; GISEL-GFX1100-NEXT: v_fma_mixhi_f16 v4, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT: v_bfe_u32 v0, v3, 0, 16
+; GISEL-GFX1100-NEXT: v_and_or_b32 v0, 0xffff0000, v4, v0
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX900-NEXT: v_max_f16_e64 v4, v3, v3 clamp
+; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX900-NEXT: v_bfe_u32 v0, v4, 0, 16
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v1, 0xffff0000
+; GISEL-GFX900-NEXT: v_and_or_b32 v0, v3, v1, v0
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: v_max_f16_e64 v4, v3, v3 clamp
+; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: v_bfe_u32 v0, v4, 0, 16
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v1, 0xffff0000
+; GISEL-GFX906-NEXT: v_and_or_b32 v0, v3, v1, v0
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v5
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_max_f16_e64 v1, v0, v0 clamp
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v2f32_clamp_postcvt_lo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v1
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_mac_f32_e32 v5, v3, v4
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v5
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_mac_f32_e32 v5, v3, v4
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
-; GISEL-CI-NEXT: v_max_f32_e32 v3, 0, v3
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v5
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v3
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GISEL-CI-NEXT: v_max_f32_e32 v1, 0, v1
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v1, v0
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
@@ -1729,37 +1781,37 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x half> %src0, <2 x half>
; SDAG-GFX1100-FAKE16-NEXT: v_mov_b32_e32 v0, v3
; SDAG-GFX1100-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; GFX900-NEXT: v_mov_b32_e32 v0, v3
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX900-NEXT: v_mov_b32_e32 v0, v3
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; GFX906-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
-; GFX906: ; %bb.0:
-; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; GFX906-NEXT: v_mov_b32_e32 v0, v3
-; GFX906-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX906-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; SDAG-GFX906: ; %bb.0:
+; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; SDAG-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v3
+; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v5, v3, v4
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v2
-; VI-NEXT: v_or_b32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v2
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
; SDAG-CI: ; %bb.0:
@@ -1790,36 +1842,92 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt_hi(<2 x half> %src0, <2 x half>
; GISEL-GFX1100-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
; GISEL-GFX1100: ; %bb.0:
; GISEL-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
-; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GISEL-GFX1100-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
-; GISEL-GFX1100-NEXT: v_mov_b32_e32 v0, v3
+; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GISEL-GFX1100-NEXT: v_fma_mixlo_f16 v4, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX1100-NEXT: v_bfe_u32 v3, v3, 0, 16
+; GISEL-GFX1100-NEXT: v_fma_mixhi_f16 v4, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX1100-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GISEL-GFX1100-NEXT: v_lshlrev_b32_e32 v0, 16, v3
+; GISEL-GFX1100-NEXT: v_and_or_b32 v0, 0xffff, v4, v0
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX900-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX900-NEXT: v_mad_mixlo_f16 v4, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GISEL-GFX900-NEXT: v_mad_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v0, 16
+; GISEL-GFX900-NEXT: v_lshlrev_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-GFX900-NEXT: v_mov_b32_e32 v1, 0xffff
+; GISEL-GFX900-NEXT: v_and_or_b32 v0, v3, v1, v0
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX906-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; GISEL-GFX906: ; %bb.0:
+; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v3, v0, v1, v2 op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: v_fma_mixlo_f16 v4, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1] clamp
+; GISEL-GFX906-NEXT: v_fma_mixhi_f16 v3, v0, v1, v2 op_sel:[1,1,1] op_sel_hi:[1,1,1]
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, 16
+; GISEL-GFX906-NEXT: v_lshlrev_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-GFX906-NEXT: v_mov_b32_e32 v1, 0xffff
+; GISEL-GFX906-NEXT: v_and_or_b32 v0, v3, v1, v0
+; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v5
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_max_f16_sdwa v1, v0, v0 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v2f32_clamp_postcvt_hi:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_mac_f32_e32 v3, v5, v4
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GISEL-CI-NEXT: v_mac_f32_e32 v5, v3, v4
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v2
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GISEL-CI-NEXT: v_max_f32_e32 v3, 0, v3
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GISEL-CI-NEXT: v_min_f32_e32 v0, 1.0, v3
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v1, v0
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v5
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-CI-NEXT: v_max_f32_e32 v1, 0, v1
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v1
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix.ll b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
index c61504d826d12..feb2a1265b889 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix.ll
@@ -9,12 +9,12 @@
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1100,GISEL-GFX1100,GISEL-GFX1100-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1100,GISEL-GFX1100,GISEL-GFX1100-FAKE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX900,GISEL-GFX900 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.06 < %s | FileCheck -check-prefixes=GFX906,GISEL-GFX906 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9 --amdhsa-code-object-version=6 < %s | FileCheck -check-prefixes=GFX9GEN,GISEL-GFX9GEN %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=CI,GISEL-CI %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1100,GISEL-GFX1100,GISEL-GFX1100-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX900,GISEL-GFX900 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.06 < %s | FileCheck -check-prefixes=GFX906,GISEL-GFX906 %s
+; RUN: llc -global-isel -mtriple=amdgpu9 --amdhsa-code-object-version=6 < %s | FileCheck -check-prefixes=GFX9GEN,GISEL-GFX9GEN %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=CI,GISEL-CI %s
define float @v_mad_mix_f32_f16lo_f16lo_f16lo(half %src0, half %src1, half %src2) #0 {
; GFX1100-LABEL: v_mad_mix_f32_f16lo_f16lo_f16lo:
@@ -169,17 +169,29 @@ define float @v_mad_mix_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x half> %
; VI-NEXT: v_mac_f32_e32 v0, v3, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_f16hi_f16hi_f16hi_elt:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v3, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; CI-NEXT: v_mac_f32_e32 v0, v3, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_f16hi_f16hi_f16hi_elt:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_f16hi_f16hi_f16hi_elt:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src2.hi = extractelement <2 x half> %src2, i32 1
@@ -1614,17 +1626,29 @@ define float @v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt(<2 x half> %src0, <2 x h
; VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; CI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-CI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_clamp_f32_f16hi_f16hi_f16hi_elt:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src2.hi = extractelement <2 x half> %src2, i32 1
@@ -2190,15 +2214,15 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo(i32 %src0.arg, half %
; VI-NEXT: v_mad_f32 v0, |v0|, v1, v2
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v3, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; CI-NEXT: v_mac_f32_e32 v0, v3, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v3, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
; GISEL-GFX1100: ; %bb.0:
@@ -2208,6 +2232,16 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo(i32 %src0.arg, half %
; GISEL-GFX1100-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, v1, v2 op_sel_hi:[1,1,1]
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_f16lo_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_mad_f32 v0, |v0|, v1, v2
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%src0 = extractelement <2 x half> %src0.arg.bc, i32 1
%src0.neg = fneg half %src0
@@ -2294,33 +2328,64 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo(i32 %src0.arg, half
; GFX906-NEXT: v_fma_mix_f32 v0, -v0, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mad_f32 v0, -v0, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v3, -v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; CI-NEXT: v_mac_f32_e32 v0, v3, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v3, -v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_f16lo_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fneg = fneg <2 x half> %src0.arg.bc
%src0 = extractelement <2 x half> %fneg, i32 1
@@ -2350,33 +2415,64 @@ define float @v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo(i32 %src0.arg, half
; GFX906-NEXT: v_fma_mix_f32 v0, |v0|, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
-; GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; VI-NEXT: v_mac_f32_e32 v0, v3, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v3, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; CI-NEXT: v_mac_f32_e32 v0, v3, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v3, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_f16lo_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
%src0 = extractelement <2 x half> %fabs, i32 1
@@ -2406,53 +2502,84 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo(i32 %src0.arg,
; GFX906-NEXT: v_fma_mix_f32 v0, -|v0|, v1, v2 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mad_f32 v0, -v0, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-GFX9GEN-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v3, -|v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v2
-; CI-NEXT: v_mac_f32_e32 v0, v3, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
- %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
- %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
- %fneg.fabs = fneg <2 x half> %fabs
- %src0 = extractelement <2 x half> %fneg.fabs, i32 1
- %src0.ext = fpext half %src0 to float
- %src1.ext = fpext half %src1 to float
- %src2.ext = fpext half %src2 to float
- %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
- ret float %result
-}
-
-define float @v_mad_mix_f32_f16lo_add_f16lo(half %src0, half %src1) {
-; GFX1100-LABEL: v_mad_mix_f32_f16lo_add_f16lo:
-; GFX1100: ; %bb.0:
-; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX1100-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,1]
-; GFX1100-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mad_f32 v0, -v0, v1, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_f16lo_add_f16lo:
-; GFX900: ; %bb.0:
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v3, -|v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; SDAG-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-GFX9GEN-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_f16lo_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v2
+; GISEL-CI-NEXT: v_mac_f32_e32 v0, v3, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
+ %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+ %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
+ %fneg.fabs = fneg <2 x half> %fabs
+ %src0 = extractelement <2 x half> %fneg.fabs, i32 1
+ %src0.ext = fpext half %src0 to float
+ %src1.ext = fpext half %src1 to float
+ %src2.ext = fpext half %src2 to float
+ %result = tail call float @llvm.fmuladd.f32(float %src0.ext, float %src1.ext, float %src2.ext)
+ ret float %result
+}
+
+define float @v_mad_mix_f32_f16lo_add_f16lo(half %src0, half %src1) {
+; GFX1100-LABEL: v_mad_mix_f32_f16lo_add_f16lo:
+; GFX1100: ; %bb.0:
+; GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX1100-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,1]
+; GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX900-LABEL: v_mad_mix_f32_f16lo_add_f16lo:
+; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX900-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
@@ -2589,15 +2716,25 @@ define float @v_mad_mix_f32_f16hi_add_f16hi_elt(<2 x half> %src0, <2 x half> %sr
; VI-NEXT: v_add_f32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_f16hi_add_f16hi_elt:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_add_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_f16hi_add_f16hi_elt:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_f16hi_add_f16hi_elt:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src0.ext = fpext half %src0.hi to float
@@ -3321,15 +3458,25 @@ define float @v_mad_mix_clamp_f32_f16hi_add_f16hi(<2 x half> %src0, <2 x half> %
; VI-NEXT: v_add_f32_e64 v0, v0, v1 clamp
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_clamp_f32_f16hi_add_f16hi:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_add_f32_e64 v0, v0, v1 clamp
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_clamp_f32_f16hi_add_f16hi:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_add_f32_e64 v0, v0, v1 clamp
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_clamp_f32_f16hi_add_f16hi:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_add_f32_e64 v0, v0, v1 clamp
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src0.ext = fpext half %src0.hi to float
@@ -3639,14 +3786,14 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_add_f16lo(i32 %src0.arg, half %sr
; VI-NEXT: v_add_f32_e64 v0, |v0|, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_add_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_add_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_add_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_add_f16lo:
; GISEL-GFX1100: ; %bb.0:
@@ -3656,6 +3803,15 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_add_f16lo(i32 %src0.arg, half %sr
; GISEL-GFX1100-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, 1.0, v1 op_sel_hi:[1,1,1]
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_add_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_add_f32_e64 v0, |v0|, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%src0 = extractelement <2 x half> %src0.arg.bc, i32 1
%src0.neg = fneg half %src0
@@ -3727,13 +3883,13 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_add_f16lo(i32 %src0.arg, half %
; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-GFX906-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
; SDAG-GFX906: ; %bb.0:
@@ -3741,30 +3897,30 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_add_f16lo(i32 %src0.arg, half %
; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, v0, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_sub_f32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_add_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
; GISEL-GFX1100: ; %bb.0:
@@ -3772,11 +3928,48 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_add_f16lo(i32 %src0.arg, half %
; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-GFX906-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
; GISEL-GFX906: ; %bb.0:
; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -v0, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_add_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fneg = fneg <2 x half> %src0.arg.bc
%src0 = extractelement <2 x half> %fneg, i32 1
@@ -3793,13 +3986,13 @@ define float @v_mad_mix_f32_preextractfabs_f16hi_add_f16lo(i32 %src0.arg, half %
; GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
; GFX906: ; %bb.0:
@@ -3807,84 +4000,121 @@ define float @v_mad_mix_f32_preextractfabs_f16hi_add_f16lo(i32 %src0.arg, half %
; GFX906-NEXT: v_fma_mix_f32 v0, |v0|, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
-;
-; VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_add_f32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_add_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
- %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
- %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
- %src0 = extractelement <2 x half> %fabs, i32 1
- %src0.ext = fpext half %src0 to float
- %src1.ext = fpext half %src1 to float
- %result = fadd float %src0.ext, %src1.ext
- ret float %result
-}
-
-define float @v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo(i32 %src0.arg, half %src1) {
-; SDAG-GFX1100-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; SDAG-GFX1100: ; %bb.0:
-; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
-; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
-; SDAG-GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-GFX900-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_add_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
+ %src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
+ %fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
+ %src0 = extractelement <2 x half> %fabs, i32 1
+ %src0.ext = fpext half %src0 to float
+ %src1.ext = fpext half %src1 to float
+ %result = fadd float %src0.ext, %src1.ext
+ ret float %result
+}
+
+define float @v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo(i32 %src0.arg, half %src1) {
+; SDAG-GFX1100-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-GFX1100: ; %bb.0:
+; SDAG-GFX1100-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
+; SDAG-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; SDAG-GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
; SDAG-GFX906: ; %bb.0:
; SDAG-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SDAG-GFX906-NEXT: v_fma_mix_f32 v0, |v0|, -1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_sub_f32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_sub_f32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_add_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
; GISEL-GFX1100: ; %bb.0:
@@ -3892,11 +4122,48 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo(i32 %src0.arg, ha
; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, -|v0|, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
; GISEL-GFX906: ; %bb.0:
; GISEL-GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-GFX906-NEXT: v_fma_mix_f32 v0, -|v0|, 1.0, v1 op_sel:[1,0,0] op_sel_hi:[1,1,1]
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_add_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_add_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
%fneg.fabs = fneg <2 x half> %fabs
@@ -4052,15 +4319,25 @@ define float @v_mad_mix_f32_f16hi_mul_f16hi_elt(<2 x half> %src0, <2 x half> %sr
; VI-NEXT: v_mul_f32_e32 v0, v0, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_f16hi_mul_f16hi_elt:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_mul_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_f16hi_mul_f16hi_elt:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_f16hi_mul_f16hi_elt:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src0.ext = fpext half %src0.hi to float
@@ -4773,15 +5050,25 @@ define float @v_mad_mix_clamp_f32_f16hi_mul_f16hi_elt(<2 x half> %src0, <2 x hal
; VI-NEXT: v_mul_f32_e64 v0, v0, v1 clamp
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_clamp_f32_f16hi_mul_f16hi_elt:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_mul_f32_e64 v0, v0, v1 clamp
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_clamp_f32_f16hi_mul_f16hi_elt:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_mul_f32_e64 v0, v0, v1 clamp
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_clamp_f32_f16hi_mul_f16hi_elt:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mul_f32_e64 v0, v0, v1 clamp
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.hi = extractelement <2 x half> %src0, i32 1
%src1.hi = extractelement <2 x half> %src1, i32 1
%src0.ext = fpext half %src0.hi to float
@@ -5003,14 +5290,14 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_mul_f16lo(i32 %src0.arg, half %sr
; VI-NEXT: v_mul_f32_e64 v0, |v0|, v1
; VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_mul_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_mul_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_mul_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX1100-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_mul_f16lo:
; GISEL-GFX1100: ; %bb.0:
@@ -5020,6 +5307,15 @@ define float @v_mad_mix_f32_precvtnegf16hi_abs_mul_f16lo(i32 %src0.arg, half %sr
; GISEL-GFX1100-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GISEL-GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, v1, neg(0) op_sel_hi:[1,1,0]
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_precvtnegf16hi_abs_mul_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mul_f32_e64 v0, |v0|, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%src0 = extractelement <2 x half> %src0.arg.bc, i32 1
%src0.neg = fneg half %src0
@@ -5091,13 +5387,13 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo(i32 %src0.arg, half %
; GFX1100-NEXT: v_fma_mix_f32 v0, -v0, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
; GFX906: ; %bb.0:
@@ -5105,30 +5401,67 @@ define float @v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo(i32 %src0.arg, half %
; GFX906-NEXT: v_fma_mix_f32 v0, -v0, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_mul_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfneg_f16hi_mul_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fneg = fneg <2 x half> %src0.arg.bc
%src0 = extractelement <2 x half> %fneg, i32 1
@@ -5145,13 +5478,13 @@ define float @v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo(i32 %src0.arg, half %
; GFX1100-NEXT: v_fma_mix_f32 v0, |v0|, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
; GFX906: ; %bb.0:
@@ -5159,30 +5492,67 @@ define float @v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo(i32 %src0.arg, half %
; GFX906-NEXT: v_fma_mix_f32 v0, |v0|, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_mul_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfabs_f16hi_mul_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
%src0 = extractelement <2 x half> %fabs, i32 1
@@ -5199,13 +5569,13 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo(i32 %src0.arg, ha
; GFX1100-NEXT: v_fma_mix_f32 v0, -|v0|, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX1100-NEXT: s_setpc_b64 s[30:31]
;
-; GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX900-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; SDAG-GFX900: ; %bb.0:
+; SDAG-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX900-NEXT: s_setpc_b64 s[30:31]
;
; GFX906-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
; GFX906: ; %bb.0:
@@ -5213,30 +5583,67 @@ define float @v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo(i32 %src0.arg, ha
; GFX906-NEXT: v_fma_mix_f32 v0, -|v0|, v1, neg(0) op_sel:[1,0,0] op_sel_hi:[1,1,0]
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
-; GFX9GEN: ; %bb.0:
-; GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; SDAG-GFX9GEN: ; %bb.0:
+; SDAG-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v0, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
-; CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
-; CI: ; %bb.0:
-; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; CI-NEXT: v_mul_f32_e32 v0, v0, v1
-; CI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; SDAG-CI: ; %bb.0:
+; SDAG-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SDAG-CI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; SDAG-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; SDAG-CI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX900-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; GISEL-GFX900: ; %bb.0:
+; GISEL-GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX900-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX900-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX900-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-GFX9GEN-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; GISEL-GFX9GEN: ; %bb.0:
+; GISEL-GFX9GEN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-GFX9GEN-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-GFX9GEN-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-GFX9GEN-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-GFX9GEN-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-VI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-CI-LABEL: v_mad_mix_f32_preextractfabsfneg_f16hi_mul_f16lo:
+; GISEL-CI: ; %bb.0:
+; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-CI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_mul_f32_e32 v0, v0, v1
+; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.arg.bc = bitcast i32 %src0.arg to <2 x half>
%fabs = call <2 x half> @llvm.fabs.v2f16(<2 x half> %src0.arg.bc)
%fneg.fabs = fneg <2 x half> %fabs
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
index 98b812d99cd5b..71b23bb18e58b 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-add.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
@@ -1168,11 +1168,20 @@ define i16 @test_vector_reduce_add_v2i16(<2 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_add_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_add_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_add_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_add_v2i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1451,13 +1460,24 @@ define i16 @test_vector_reduce_add_v4i16(<4 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_add_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_add_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_add_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_add_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1620,17 +1640,32 @@ define i16 @test_vector_reduce_add_v8i16(<8 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_add_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_add_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_add_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_add_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_u16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_add_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1849,25 +1884,48 @@ define i16 @test_vector_reduce_add_v16i16(<16 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_add_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v2, v2, v6
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_add_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_add_u16_e32 v4, v11, v10
-; GFX8-NEXT: v_add_u16_e32 v5, v9, v8
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_add_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_add_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_add_u16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_add_u16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_add_u16_e32 v4, v11, v10
+; GFX8-SDAG-NEXT: v_add_u16_e32 v5, v9, v8
+; GFX8-SDAG-NEXT: v_add_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_add_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_u16_e32 v8, v0, v4
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v4, v1, v5
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v5, v2, v6
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v6, v3, v7
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, v4, v6
+; GFX8-GISEL-NEXT: v_add_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_add_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
index 37d3317ee9dd2..63a3be6faf782 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fadd.ll
@@ -1,35 +1,49 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define half @test_vector_reduce_fadd_v2half(half %sp, <2 x half> %v) {
-; GFX7-LABEL: test_vector_reduce_fadd_v2half:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_fadd_v2half:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_fadd_v2half:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GISEL-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_fadd_v2half:
; GFX8: ; %bb.0: ; %entry
@@ -73,8 +87,9 @@ define half @test_vector_reduce_fadd_v2half(half %sp, <2 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v2half:
@@ -119,8 +134,9 @@ define half @test_vector_reduce_fadd_v2half(half %sp, <2 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v2half:
@@ -399,11 +415,13 @@ define half @test_vector_reduce_fadd_v4half(half %sp, <4 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v4half:
@@ -459,11 +477,13 @@ define half @test_vector_reduce_fadd_v4half(half %sp, <4 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v4half:
@@ -651,17 +671,21 @@ define half @test_vector_reduce_fadd_v8half(half %sp, <8 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v8half:
@@ -739,17 +763,21 @@ define half @test_vector_reduce_fadd_v8half(half %sp, <8 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v8half:
@@ -1069,29 +1097,37 @@ define half @test_vector_reduce_fadd_v16half(half %sp, <16 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.h
+; GFX11-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v16half:
@@ -1213,29 +1249,37 @@ define half @test_vector_reduce_fadd_v16half(half %sp, <16 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v2.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v3.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v4.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v5.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v6.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v7.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v8.h
+; GFX12-GISEL-TRUE16-NEXT: v_add_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fadd_v16half:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
index b5d61a9728d03..47e9160fdfac2 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-fmul.ll
@@ -1,35 +1,49 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define half @test_vector_reduce_fmul_v2half(half %sp, <2 x half> %v) {
-; GFX7-LABEL: test_vector_reduce_fmul_v2half:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v2
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_fmul_v2half:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_fmul_v2half:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_fmul_v2half:
; GFX8: ; %bb.0: ; %entry
@@ -73,8 +87,9 @@ define half @test_vector_reduce_fmul_v2half(half %sp, <2 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v2half:
@@ -119,8 +134,9 @@ define half @test_vector_reduce_fmul_v2half(half %sp, <2 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v2half:
@@ -399,11 +415,13 @@ define half @test_vector_reduce_fmul_v4half(half %sp, <4 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v4half:
@@ -459,11 +477,13 @@ define half @test_vector_reduce_fmul_v4half(half %sp, <4 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v4half:
@@ -651,17 +671,21 @@ define half @test_vector_reduce_fmul_v8half(half %sp, <8 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v8half:
@@ -739,17 +763,21 @@ define half @test_vector_reduce_fmul_v8half(half %sp, <8 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v8half:
@@ -1069,29 +1097,37 @@ define half @test_vector_reduce_fmul_v16half(half %sp, <16 x half> %v) {
; GFX11-GISEL-TRUE16: ; %bb.0: ; %entry
; GFX11-GISEL-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.l
-; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.h
+; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX11-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.l
; GFX11-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.h
+; GFX11-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX11-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v16half:
@@ -1213,29 +1249,37 @@ define half @test_vector_reduce_fmul_v16half(half %sp, <16 x half> %v) {
; GFX12-GISEL-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.h
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v2.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v3.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v4.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v5
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v5.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v6
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v6.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v7
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.l
-; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v7.h
+; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
+; GFX12-GISEL-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v8
; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.l
; GFX12-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v8.h
+; GFX12-GISEL-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.l, v1.l
; GFX12-GISEL-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-FAKE16-LABEL: test_vector_reduce_fmul_v16half:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
index 4190a350e70f0..db873d5385dbc 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
@@ -985,18 +985,11 @@ define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) {
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-SDAG-LABEL: test_vector_reduce_mul_v2i16:
-; GFX8-SDAG: ; %bb.0: ; %entry
-; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-GISEL-LABEL: test_vector_reduce_mul_v2i16:
-; GFX8-GISEL: ; %bb.0: ; %entry
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
-; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+; GFX8-LABEL: test_vector_reduce_mul_v2i16:
+; GFX8: ; %bb.0: ; %entry
+; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_vector_reduce_mul_v2i16:
; GFX9: ; %bb.0: ; %entry
@@ -1239,7 +1232,6 @@ define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) {
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
-; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_vector_reduce_mul_v4i16:
@@ -1386,7 +1378,6 @@ define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) {
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
-; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v8i16:
@@ -1618,7 +1609,6 @@ define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) {
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v5, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
-; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v16i16:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index 97668d8cde573..e225d5ca07e05 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -1,21 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
; FIXME: codegen regression, related to:
; - looking through sgpr to vgpr copy
@@ -276,41 +276,89 @@ entry:
}
define i8 @test_vector_reduce_or_v4i8(<4 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v4i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v4i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v4i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v4i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff00, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff0000, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff000000, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v4i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v4i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v4i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v4i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0xff00
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, 0xff0000, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, 0xff000000, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v4i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v4i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff00
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff000000
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v4i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v4i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xff00
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -336,10 +384,13 @@ define i8 @test_vector_reduce_or_v4i8(<4 x i8> %v) {
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v3, 0xff00
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v4i8:
@@ -378,10 +429,13 @@ define i8 @test_vector_reduce_or_v4i8(<4 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, 0xff00
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.or.v4i8(<4 x i8> %v)
@@ -389,57 +443,119 @@ entry:
}
define i8 @test_vector_reduce_or_v8i8(<8 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v8i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v8i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v8i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v8i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff00, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff0000, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff000000, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v8i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v8i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v8i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v8i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0xff00
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, 0xff0000, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, 0xff000000, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v8i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v8i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff00
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff000000
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v8i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v8i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xff00
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -474,16 +590,17 @@ define i8 @test_vector_reduce_or_v8i8(<8 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_or_v8i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v3, 0xff00
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v8i8:
@@ -531,16 +648,17 @@ define i8 @test_vector_reduce_or_v8i8(<8 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v3, 0xff00
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.or.v8i8(<8 x i8> %v)
@@ -548,89 +666,177 @@ entry:
}
define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v16i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX7-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX7-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v16i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v16i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v16i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX7-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX7-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX7-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff00, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff0000, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, 0xff000000, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_or_v16i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX9-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX9-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX9-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX9-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX9-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v16i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_or_v16i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX10-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX10-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v11
-; GFX10-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX10-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX10-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v16i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX8-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX8-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0xff00
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, 0xff0000, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, 0xff000000, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_or_v16i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX9-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX9-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX9-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX9-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX9-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX9-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_or_v16i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX9-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX9-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX9-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX9-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX9-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX9-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff00
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xff0000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff000000
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_or_v16i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX10-SDAG-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX10-SDAG-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, v2, v10
+; GFX10-SDAG-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX10-SDAG-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX10-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX10-SDAG-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX10-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_or_v16i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
+; GFX10-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GFX10-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; GFX10-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX10-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX10-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX10-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX10-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff00
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX10-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -683,25 +889,23 @@ define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_or_v16i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
; GFX11-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX11-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
; GFX11-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX11-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX11-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX11-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX11-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX11-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX11-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0xff00
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX11-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_or_v16i8:
@@ -767,25 +971,23 @@ define i8 @test_vector_reduce_or_v16i8(<16 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
; GFX12-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
-; GFX12-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
-; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
; GFX12-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX12-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX12-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX12-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; GFX12-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; GFX12-GISEL-NEXT: v_or3_b32 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v8, v4
+; GFX12-GISEL-NEXT: v_or3_b32 v2, v2, v10, v6
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0xff00
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX12-GISEL-NEXT: v_mov_b32_e32 v1, 0xff000000
+; GFX12-GISEL-NEXT: v_or3_b32 v0, 0xff0000, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.or.v16i8(<16 x i8> %v)
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
index 26e542c4fccb7..659882d67d6c8 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smax.ll
@@ -1,21 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_smax_v2i8(<2 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_smax_v2i8:
@@ -296,50 +296,145 @@ entry:
}
define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v4i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX7-NEXT: v_max3_i32 v0, v0, v2, v1
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v4i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v2, v1
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v4i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v4i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, s4, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_max3_i32 v2, v2, s4, s4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-GISEL-NEXT: v_max3_i32 v3, v3, s4, s4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smax_v4i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-NEXT: v_max3_i16 v0, v0, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v4i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smax_v4i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-NEXT: v_max_i16 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_max3_i16 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v4i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v2, sext(v2), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v3, sext(v3), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, s4, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_smax_v4i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-SDAG-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_smax_v4i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v1, sext(v1), v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_max_i16_e32 v1, s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_max3_i16 v2, v2, s0, s0
+; GFX9-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smax_v4i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v1, v1, v3
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smax_v4i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-GISEL-NEXT: v_max_i16 v1, v1, v3
+; GFX10-GISEL-NEXT: v_max3_i16 v5, v2, s4, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v3, v3, s4, s4
+; GFX10-GISEL-NEXT: v_max_i16 v6, v1, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -380,18 +475,29 @@ define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smax_v4i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX11-GISEL-NEXT: s_sext_i32_i8 s0, s0
; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_max_i16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX11-GISEL-NEXT: v_max3_i16 v5, v2, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_max_i16 v4, v1, s0
; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v4i8:
@@ -445,18 +551,29 @@ define i8 @test_vector_reduce_smax_v4i8(<4 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX12-GISEL-NEXT: s_sext_i32_i8 s0, s0
; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_max_i16 v1, v1, v3
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX12-GISEL-NEXT: v_max3_i16 v5, v2, s0, s0
+; GFX12-GISEL-NEXT: v_max_i16 v4, v1, s0
; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.smax.v4i8(<4 x i8> %v)
@@ -464,73 +581,189 @@ entry:
}
define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v8i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX7-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v4
-; GFX7-NEXT: v_max_i32_e32 v1, v1, v5
-; GFX7-NEXT: v_max3_i32 v1, v1, v3, v7
-; GFX7-NEXT: v_max3_i32 v0, v0, v2, v6
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v8i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_max_i32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_max3_i32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v8i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v8i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v7, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v3, v3, v6
+; GFX7-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_max3_i32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, s4, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_max3_i32 v2, v2, s4, s4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-GISEL-NEXT: v_max3_i32 v3, v3, s4, s4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smax_v8i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX9-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX9-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX9-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX9-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_max3_i16 v0, v0, v2, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v8i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smax_v8i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX10-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-NEXT: v_max_i16 v1, v1, v5
-; GFX10-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX10-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX10-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_max_i16 v0, v0, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_max3_i16 v0, v0, v2, v4
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_max_i16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v8i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, s4, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_smax_v8i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_i16 v1, v1, v3, v7
+; GFX9-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max3_i16 v0, v0, v2, v6
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_smax_v8i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-GISEL-NEXT: v_max3_i16 v0, v0, v4, v2
+; GFX9-GISEL-NEXT: v_max3_i16 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_max_i16_e32 v1, s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_max3_i16 v2, v2, s0, s0
+; GFX9-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smax_v8i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v1, v1, v5
+; GFX10-SDAG-NEXT: v_max3_i16 v1, v1, v3, v7
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v4, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v4, v6, 0, 8
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_max_i16 v0, v0, v3
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-SDAG-NEXT: v_max3_i16 v0, v0, v2, v4
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smax_v8i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-GISEL-NEXT: v_max_i16 v3, v3, v7
+; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX10-GISEL-NEXT: v_max_i16 v2, v2, v6
+; GFX10-GISEL-NEXT: v_max3_i16 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-GISEL-NEXT: v_max3_i16 v3, v3, s4, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: v_max_i16 v4, v1, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v2, v2, s4, s4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -599,26 +832,36 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smax_v8i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-GISEL-NEXT: v_max_i16 v1, v1, v5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX11-GISEL-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_max_i16 v0, v0, v3
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v2, v4
+; GFX11-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-GISEL-NEXT: v_max_i16 v3, v3, v7
+; GFX11-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX11-GISEL-NEXT: v_max_i16 v2, v2, v6
+; GFX11-GISEL-NEXT: v_max3_i16 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: v_max_i16 v5, v1, s0
+; GFX11-GISEL-NEXT: v_max3_i16 v2, v2, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v8i8:
@@ -700,26 +943,37 @@ define i8 @test_vector_reduce_smax_v8i8(<8 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-GISEL-NEXT: v_max_i16 v1, v1, v5
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-GISEL-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX12-GISEL-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_max_i16 v0, v0, v3
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v2, v4
+; GFX12-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX12-GISEL-NEXT: v_max_i16 v3, v3, v7
+; GFX12-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX12-GISEL-NEXT: v_max_i16 v2, v2, v6
+; GFX12-GISEL-NEXT: v_max3_i16 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: v_max_i16 v5, v1, s0
+; GFX12-GISEL-NEXT: v_max3_i16 v2, v2, s0, s0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> %v)
@@ -727,115 +981,274 @@ entry:
}
define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v16i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX7-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX7-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX7-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX7-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX7-NEXT: v_bfe_i32 v15, v15, 0, 8
-; GFX7-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX7-NEXT: v_bfe_i32 v10, v10, 0, 8
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-NEXT: v_bfe_i32 v14, v14, 0, 8
-; GFX7-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX7-NEXT: v_max_i32_e32 v1, v1, v9
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v8
-; GFX7-NEXT: v_max_i32_e32 v6, v6, v14
-; GFX7-NEXT: v_max_i32_e32 v2, v2, v10
-; GFX7-NEXT: v_max_i32_e32 v7, v7, v15
-; GFX7-NEXT: v_max_i32_e32 v3, v3, v11
-; GFX7-NEXT: v_max3_i32 v0, v0, v4, v12
-; GFX7-NEXT: v_max3_i32 v1, v1, v5, v13
-; GFX7-NEXT: v_max3_i32 v1, v1, v3, v7
-; GFX7-NEXT: v_max3_i32 v0, v0, v2, v6
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v16i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v14, v14, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-SDAG-NEXT: v_max_i32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_max_i32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_max_i32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_max_i32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_max_i32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v4, v12
+; GFX7-SDAG-NEXT: v_max3_i32 v1, v1, v5, v13
+; GFX7-SDAG-NEXT: v_max3_i32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v16i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_i16_e32 v3, v3, v7
-; GFX8-NEXT: v_max_i16_e32 v1, v1, v5
-; GFX8-NEXT: v_max_i16_e32 v2, v2, v6
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v4
-; GFX8-NEXT: v_max_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v16i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v13, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v5, v5, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v14, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v6, v6, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v15, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX7-GISEL-NEXT: v_max_i32_e32 v7, v7, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX7-GISEL-NEXT: v_max3_i32 v1, v1, v9, v5
+; GFX7-GISEL-NEXT: v_max3_i32 v3, v3, v11, v7
+; GFX7-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v8, v4
+; GFX7-GISEL-NEXT: v_max3_i32 v2, v2, v10, v6
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, s4, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_max3_i32 v2, v2, s4, s4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-GISEL-NEXT: v_max3_i32 v3, v3, s4, s4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smax_v16i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX9-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX9-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_i16 v1, v1, v5, v13
-; GFX9-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX9-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX9-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX9-NEXT: v_max_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_i16 v0, v0, v4, v12
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_max3_i16 v0, v0, v2, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v16i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_i16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_max_i16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_max_i16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smax_v16i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX10-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX10-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX10-NEXT: v_bfe_i32 v15, v15, 0, 8
-; GFX10-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX10-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-NEXT: v_max_i16 v1, v1, v9
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-NEXT: v_bfe_i32 v9, v12, 0, 8
-; GFX10-NEXT: v_max_i16 v7, v7, v15
-; GFX10-NEXT: v_max_i16 v3, v3, v11
-; GFX10-NEXT: v_max3_i16 v1, v1, v5, v13
-; GFX10-NEXT: v_bfe_i32 v5, v8, 0, 8
-; GFX10-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX10-NEXT: v_bfe_i32 v8, v10, 0, 8
-; GFX10-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX10-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX10-NEXT: v_bfe_i32 v3, v14, 0, 8
-; GFX10-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-NEXT: v_max_i16 v0, v0, v5
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_max_i16 v3, v6, v3
-; GFX10-NEXT: v_max_i16 v2, v2, v8
-; GFX10-NEXT: v_max3_i16 v0, v0, v4, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_max3_i16 v0, v0, v2, v3
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_max_i16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v16i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_max_i16_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, s4, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_smax_v16i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_i16 v1, v1, v5, v13
+; GFX9-SDAG-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_i16 v1, v1, v3, v7
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_i16 v0, v0, v4, v12
+; GFX9-SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max3_i16 v0, v0, v2, v6
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_smax_v16i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max3_i16 v1, v1, v9, v5
+; GFX9-GISEL-NEXT: v_max3_i16 v3, v3, v11, v7
+; GFX9-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-GISEL-NEXT: v_max3_i16 v0, v0, v8, v4
+; GFX9-GISEL-NEXT: v_max3_i16 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_max_i16_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_max_i16_e32 v1, s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_max3_i16 v2, v2, s0, s0
+; GFX9-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smax_v16i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v1, v1, v9
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v7, v7, v15
+; GFX10-SDAG-NEXT: v_max_i16 v3, v3, v11
+; GFX10-SDAG-NEXT: v_max3_i16 v1, v1, v5, v13
+; GFX10-SDAG-NEXT: v_bfe_i32 v5, v8, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v8, v10, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-SDAG-NEXT: v_max3_i16 v1, v1, v3, v7
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v14, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v0, v0, v5
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_max_i16 v3, v6, v3
+; GFX10-SDAG-NEXT: v_max_i16 v2, v2, v8
+; GFX10-SDAG-NEXT: v_max3_i16 v0, v0, v4, v9
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-SDAG-NEXT: v_max3_i16 v0, v0, v2, v3
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_max_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smax_v16i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX10-GISEL-NEXT: v_max_i16 v5, v5, v13
+; GFX10-GISEL-NEXT: v_max_i16 v7, v7, v15
+; GFX10-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v13, v14, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX10-GISEL-NEXT: v_max3_i16 v1, v1, v9, v5
+; GFX10-GISEL-NEXT: v_max3_i16 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_bfe_i32 v5, v10, 0, 8
+; GFX10-GISEL-NEXT: v_max_i16 v6, v6, v13
+; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v7, v8, 0, 8
+; GFX10-GISEL-NEXT: v_max_i16 v4, v4, v12
+; GFX10-GISEL-NEXT: v_max_i16 v1, v1, v3
+; GFX10-GISEL-NEXT: v_max3_i16 v2, v2, v5, v6
+; GFX10-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-GISEL-NEXT: v_max3_i16 v0, v0, v7, v4
+; GFX10-GISEL-NEXT: v_max_i16 v4, v1, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v6, v2, s4, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v3, v3, s4, s4
+; GFX10-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -939,40 +1352,49 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smax_v16i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX11-GISEL-NEXT: v_max_i16 v1, v1, v9
-; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX11-GISEL-NEXT: v_max_i16 v5, v5, v13
; GFX11-GISEL-NEXT: v_max_i16 v7, v7, v15
-; GFX11-GISEL-NEXT: v_max_i16 v3, v3, v11
-; GFX11-GISEL-NEXT: v_max3_i16 v1, v1, v5, v13
-; GFX11-GISEL-NEXT: v_bfe_i32 v5, v8, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v8, v10, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX11-GISEL-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX11-GISEL-NEXT: v_bfe_i32 v3, v14, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-GISEL-NEXT: v_max_i16 v0, v0, v5
+; GFX11-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX11-GISEL-NEXT: v_max3_i16 v1, v1, v9, v5
+; GFX11-GISEL-NEXT: v_max3_i16 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: v_bfe_i32 v5, v14, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX11-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX11-GISEL-NEXT: v_max_i16 v1, v1, v3
+; GFX11-GISEL-NEXT: v_max_i16 v5, v6, v5
+; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v6, v8, 0, 8
+; GFX11-GISEL-NEXT: v_max_i16 v4, v4, v9
+; GFX11-GISEL-NEXT: v_max_i16 v8, v1, s0
+; GFX11-GISEL-NEXT: v_max3_i16 v2, v2, v7, v5
+; GFX11-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v6, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v8
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: v_max_i16 v3, v6, v3
+; GFX11-GISEL-NEXT: v_max3_i16 v5, v2, s0, s0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_max_i16 v2, v2, v8
-; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v4, v9
+; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v4
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: v_max3_i16 v0, v0, v2, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smax_v16i8:
@@ -1089,40 +1511,50 @@ define i8 @test_vector_reduce_smax_v16i8(<16 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX12-GISEL-NEXT: v_max_i16 v1, v1, v9
-; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX12-GISEL-NEXT: v_max_i16 v5, v5, v13
; GFX12-GISEL-NEXT: v_max_i16 v7, v7, v15
-; GFX12-GISEL-NEXT: v_max_i16 v3, v3, v11
-; GFX12-GISEL-NEXT: v_max3_i16 v1, v1, v5, v13
-; GFX12-GISEL-NEXT: v_bfe_i32 v5, v8, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v8, v10, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX12-GISEL-NEXT: v_max3_i16 v1, v1, v3, v7
-; GFX12-GISEL-NEXT: v_bfe_i32 v3, v14, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-GISEL-NEXT: v_max_i16 v0, v0, v5
+; GFX12-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX12-GISEL-NEXT: v_max3_i16 v1, v1, v9, v5
+; GFX12-GISEL-NEXT: v_max3_i16 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: v_bfe_i32 v5, v14, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX12-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX12-GISEL-NEXT: v_max_i16 v1, v1, v3
+; GFX12-GISEL-NEXT: v_max_i16 v5, v6, v5
+; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v6, v8, 0, 8
+; GFX12-GISEL-NEXT: v_max_i16 v4, v4, v9
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_max_i16 v8, v1, s0
+; GFX12-GISEL-NEXT: v_max3_i16 v2, v2, v7, v5
+; GFX12-GISEL-NEXT: v_max3_i16 v3, v3, s0, s0
+; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v6, v4
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: v_max_i16 v3, v6, v3
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v8
+; GFX12-GISEL-NEXT: v_max3_i16 v5, v2, s0, s0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_max_i16 v2, v2, v8
-; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v4, v9
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v2, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: v_max_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: v_max3_i16 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %v)
@@ -1131,19 +1563,41 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smax_v2i16(<2 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v2i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v2i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v2i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, v1, v0
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smax_v2i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1372,24 +1826,51 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smax_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v3, 16, v1
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-NEXT: v_max3_i32 v0, v0, v1, v2
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v3, 16, v1
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v1, v2
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_max3_i32 v1, v2, v3, v0
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smax_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1494,35 +1975,73 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smax_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v5, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v7, 16, v2
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v4, 16, v1
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v6, 16, v3
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX7-NEXT: v_max_i32_e32 v5, v5, v7
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v2
-; GFX7-NEXT: v_max3_i32 v2, v5, v4, v6
-; GFX7-NEXT: v_max3_i32 v0, v0, v1, v3
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v5, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v7, 16, v2
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v4, 16, v1
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v6, 16, v3
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX7-SDAG-NEXT: v_max_i32_e32 v5, v5, v7
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_max3_i32 v2, v5, v4, v6
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v1, v3
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_e32 v1, v1, v3
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_i16_e32 v2, v5, v4
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v7, v3, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v2, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX7-GISEL-NEXT: v_max_i32_e32 v6, v6, v7
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_max3_i32 v3, v4, v5, v6
+; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_max_i32_e32 v1, v3, v0
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_i16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_i16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smax_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1653,57 +2172,116 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smax_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smax_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v9, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v13, 16, v4
-; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v8, 16, v2
-; GFX7-NEXT: v_ashrrev_i32_e32 v10, 16, v1
-; GFX7-NEXT: v_ashrrev_i32_e32 v11, 16, v3
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v12, 16, v6
-; GFX7-NEXT: v_ashrrev_i32_e32 v14, 16, v5
-; GFX7-NEXT: v_ashrrev_i32_e32 v15, 16, v7
-; GFX7-NEXT: v_bfe_i32 v6, v6, 0, 16
-; GFX7-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX7-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX7-NEXT: v_max_i32_e32 v9, v9, v13
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v4
-; GFX7-NEXT: v_max_i32_e32 v11, v11, v15
-; GFX7-NEXT: v_max_i32_e32 v10, v10, v14
-; GFX7-NEXT: v_max_i32_e32 v3, v3, v7
-; GFX7-NEXT: v_max_i32_e32 v1, v1, v5
-; GFX7-NEXT: v_max3_i32 v4, v9, v8, v12
-; GFX7-NEXT: v_max3_i32 v0, v0, v2, v6
-; GFX7-NEXT: v_max3_i32 v2, v4, v10, v11
-; GFX7-NEXT: v_max3_i32 v0, v0, v1, v3
-; GFX7-NEXT: v_max_i32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smax_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v9, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v13, 16, v4
+; GFX7-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v8, 16, v2
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v10, 16, v1
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v11, 16, v3
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v12, 16, v6
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v14, 16, v5
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v15, 16, v7
+; GFX7-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 16
+; GFX7-SDAG-NEXT: v_max_i32_e32 v9, v9, v13
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_max_i32_e32 v11, v11, v15
+; GFX7-SDAG-NEXT: v_max_i32_e32 v10, v10, v14
+; GFX7-SDAG-NEXT: v_max_i32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_max_i32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_max3_i32 v4, v9, v8, v12
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_max3_i32 v2, v4, v10, v11
+; GFX7-SDAG-NEXT: v_max3_i32 v0, v0, v1, v3
+; GFX7-SDAG-NEXT: v_max_i32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smax_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_i16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_e32 v2, v2, v6
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v4
-; GFX8-NEXT: v_max_i16_e32 v3, v3, v7
-; GFX8-NEXT: v_max_i16_e32 v1, v1, v5
-; GFX8-NEXT: v_max_i16_e32 v4, v11, v10
-; GFX8-NEXT: v_max_i16_e32 v5, v9, v8
-; GFX8-NEXT: v_max_i16_e32 v1, v1, v3
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_i16_e32 v2, v5, v4
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smax_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v2, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v13, v6, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v6, 16, 16
+; GFX7-GISEL-NEXT: v_max_i32_e32 v12, v12, v13
+; GFX7-GISEL-NEXT: v_max_i32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v3, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v13, v7, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v7, v7, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v8, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v9, v4, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v4, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v10, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v11, v5, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v5, 16, 16
+; GFX7-GISEL-NEXT: v_max_i32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_max_i32_e32 v6, v6, v13
+; GFX7-GISEL-NEXT: v_max3_i32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_max3_i32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_max3_i32 v7, v8, v9, v12
+; GFX7-GISEL-NEXT: v_max3_i32 v2, v10, v11, v6
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_max3_i32 v1, v7, v2, v0
+; GFX7-GISEL-NEXT: v_max_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smax_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_i16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_max_i16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_max_i16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_max_i16_e32 v4, v11, v10
+; GFX8-SDAG-NEXT: v_max_i16_e32 v5, v9, v8
+; GFX8-SDAG-NEXT: v_max_i16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_i16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smax_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_i16_e32 v8, v0, v4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v4, v1, v5
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v5, v2, v6
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v6, v3, v7
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, v4, v6
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_max_i16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_max_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_i16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smax_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
index 6510599f0b9b4..09f08d332793e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-smin.ll
@@ -1,21 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_smin_v2i8(<2 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_smin_v2i8:
@@ -296,50 +296,145 @@ entry:
}
define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v4i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX7-NEXT: v_min3_i32 v0, v0, v2, v1
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v4i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v2, v1
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v4i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v4i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, s4, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_min3_i32 v2, v2, s4, s4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-GISEL-NEXT: v_min3_i32 v3, v3, s4, s4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smin_v4i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX9-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX9-NEXT: v_min3_i16 v0, v0, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v4i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smin_v4i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-NEXT: v_min_i16 v1, v1, v3
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_min3_i16 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v4i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v2, sext(v2), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v3, sext(v3), v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, s4, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_smin_v4i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v3) dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-SDAG-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_smin_v4i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v1, sext(v1), v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_min_i16_e32 v1, s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_min3_i16 v2, v2, s0, s0
+; GFX9-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smin_v4i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v1, v1, v3
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smin_v4i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-GISEL-NEXT: v_min_i16 v1, v1, v3
+; GFX10-GISEL-NEXT: v_min3_i16 v5, v2, s4, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v3, v3, s4, s4
+; GFX10-GISEL-NEXT: v_min_i16 v6, v1, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -380,18 +475,29 @@ define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smin_v4i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX11-GISEL-NEXT: s_sext_i32_i8 s0, s0
; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_min_i16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX11-GISEL-NEXT: v_min3_i16 v5, v2, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_min_i16 v4, v1, s0
; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v4i8:
@@ -445,18 +551,29 @@ define i8 @test_vector_reduce_smin_v4i8(<4 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX12-GISEL-NEXT: s_sext_i32_i8 s0, s0
; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_min_i16 v1, v1, v3
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX12-GISEL-NEXT: v_min3_i16 v5, v2, s0, s0
+; GFX12-GISEL-NEXT: v_min_i16 v4, v1, s0
; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.smin.v4i8(<4 x i8> %v)
@@ -464,73 +581,189 @@ entry:
}
define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v8i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX7-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v4
-; GFX7-NEXT: v_min_i32_e32 v1, v1, v5
-; GFX7-NEXT: v_min3_i32 v1, v1, v3, v7
-; GFX7-NEXT: v_min3_i32 v0, v0, v2, v6
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v8i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_min_i32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_min3_i32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v8i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v8i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v7, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v3, v3, v6
+; GFX7-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_min3_i32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, s4, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_min3_i32 v2, v2, s4, s4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-GISEL-NEXT: v_min3_i32 v3, v3, s4, s4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smin_v8i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX9-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX9-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX9-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX9-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX9-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_min3_i16 v0, v0, v2, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v8i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smin_v8i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX10-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-NEXT: v_min_i16 v1, v1, v5
-; GFX10-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX10-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX10-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_min_i16 v0, v0, v3
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_min3_i16 v0, v0, v2, v4
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_min_i16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v8i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, s4, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_smin_v8i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v5) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min3_i16 v1, v1, v3, v7
+; GFX9-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_min3_i16 v0, v0, v2, v6
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_smin_v8i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v7) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-GISEL-NEXT: v_min3_i16 v0, v0, v4, v2
+; GFX9-GISEL-NEXT: v_min3_i16 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_min_i16_e32 v1, s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_min3_i16 v2, v2, s0, s0
+; GFX9-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smin_v8i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v1, v1, v5
+; GFX10-SDAG-NEXT: v_min3_i16 v1, v1, v3, v7
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v4, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v4, v6, 0, 8
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_min_i16 v0, v0, v3
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-SDAG-NEXT: v_min3_i16 v0, v0, v2, v4
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smin_v8i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-GISEL-NEXT: v_min_i16 v3, v3, v7
+; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX10-GISEL-NEXT: v_min_i16 v2, v2, v6
+; GFX10-GISEL-NEXT: v_min3_i16 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-GISEL-NEXT: v_min3_i16 v3, v3, s4, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: v_min_i16 v4, v1, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v2, v2, s4, s4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -599,26 +832,36 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smin_v8i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-GISEL-NEXT: v_min_i16 v1, v1, v5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX11-GISEL-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_min_i16 v0, v0, v3
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v2, v4
+; GFX11-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX11-GISEL-NEXT: v_min_i16 v3, v3, v7
+; GFX11-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX11-GISEL-NEXT: v_min_i16 v2, v2, v6
+; GFX11-GISEL-NEXT: v_min3_i16 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: v_min_i16 v5, v1, s0
+; GFX11-GISEL-NEXT: v_min3_i16 v2, v2, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v8i8:
@@ -700,26 +943,37 @@ define i8 @test_vector_reduce_smin_v8i8(<8 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-GISEL-NEXT: v_min_i16 v1, v1, v5
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-GISEL-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX12-GISEL-NEXT: v_bfe_i32 v3, v4, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v4, v6, 0, 8
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_min_i16 v0, v0, v3
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v2, v4
+; GFX12-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX12-GISEL-NEXT: v_min_i16 v3, v3, v7
+; GFX12-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX12-GISEL-NEXT: v_min_i16 v2, v2, v6
+; GFX12-GISEL-NEXT: v_min3_i16 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: v_min_i16 v5, v1, s0
+; GFX12-GISEL-NEXT: v_min3_i16 v2, v2, s0, s0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> %v)
@@ -727,115 +981,274 @@ entry:
}
define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v16i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_bfe_i32 v8, v8, 0, 8
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX7-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX7-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX7-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX7-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX7-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX7-NEXT: v_bfe_i32 v15, v15, 0, 8
-; GFX7-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX7-NEXT: v_bfe_i32 v10, v10, 0, 8
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX7-NEXT: v_bfe_i32 v14, v14, 0, 8
-; GFX7-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX7-NEXT: v_min_i32_e32 v1, v1, v9
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v8
-; GFX7-NEXT: v_min_i32_e32 v6, v6, v14
-; GFX7-NEXT: v_min_i32_e32 v2, v2, v10
-; GFX7-NEXT: v_min_i32_e32 v7, v7, v15
-; GFX7-NEXT: v_min_i32_e32 v3, v3, v11
-; GFX7-NEXT: v_min3_i32 v0, v0, v4, v12
-; GFX7-NEXT: v_min3_i32 v1, v1, v5, v13
-; GFX7-NEXT: v_min3_i32 v1, v1, v3, v7
-; GFX7-NEXT: v_min3_i32 v0, v0, v2, v6
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v16i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v14, v14, 0, 8
+; GFX7-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-SDAG-NEXT: v_min_i32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_min_i32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_min_i32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_min_i32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_min_i32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v4, v12
+; GFX7-SDAG-NEXT: v_min3_i32 v1, v1, v5, v13
+; GFX7-SDAG-NEXT: v_min3_i32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v16i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_e32 v3, v3, v7
-; GFX8-NEXT: v_min_i16_e32 v1, v1, v5
-; GFX8-NEXT: v_min_i16_e32 v2, v2, v6
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v4
-; GFX8-NEXT: v_min_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v16i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v13, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v5, v5, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v14, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v6, v6, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v15, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX7-GISEL-NEXT: v_min_i32_e32 v7, v7, v12
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX7-GISEL-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX7-GISEL-NEXT: v_min3_i32 v1, v1, v9, v5
+; GFX7-GISEL-NEXT: v_min3_i32 v3, v3, v11, v7
+; GFX7-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v8, v4
+; GFX7-GISEL-NEXT: v_min3_i32 v2, v2, v10, v6
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, s4, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_min3_i32 v2, v2, s4, s4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX7-GISEL-NEXT: v_min3_i32 v3, v3, s4, s4
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_smin_v16i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX9-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX9-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min3_i16 v1, v1, v5, v13
-; GFX9-NEXT: v_bfe_i32 v12, v12, 0, 8
-; GFX9-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX9-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX9-NEXT: v_min_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_min3_i16 v0, v0, v4, v12
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_min3_i16 v0, v0, v2, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v16i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_min_i16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_min_i16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_min_i16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_smin_v16i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_bfe_i32 v13, v13, 0, 8
-; GFX10-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX10-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX10-NEXT: v_bfe_i32 v15, v15, 0, 8
-; GFX10-NEXT: v_bfe_i32 v7, v7, 0, 8
-; GFX10-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX10-NEXT: v_min_i16 v1, v1, v9
-; GFX10-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX10-NEXT: v_bfe_i32 v9, v12, 0, 8
-; GFX10-NEXT: v_min_i16 v7, v7, v15
-; GFX10-NEXT: v_min_i16 v3, v3, v11
-; GFX10-NEXT: v_min3_i16 v1, v1, v5, v13
-; GFX10-NEXT: v_bfe_i32 v5, v8, 0, 8
-; GFX10-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX10-NEXT: v_bfe_i32 v8, v10, 0, 8
-; GFX10-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX10-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX10-NEXT: v_bfe_i32 v3, v14, 0, 8
-; GFX10-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX10-NEXT: v_min_i16 v0, v0, v5
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_min_i16 v3, v6, v3
-; GFX10-NEXT: v_min_i16 v2, v2, v8
-; GFX10-NEXT: v_min3_i16 v0, v0, v4, v9
-; GFX10-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX10-NEXT: v_min3_i16 v0, v0, v2, v3
-; GFX10-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX10-NEXT: v_min_i16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v16i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_min_i16_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, s4, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, s4, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_min_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v3, s4, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_smin_v16i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v1, sext(v1), sext(v9) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v3, sext(v3), sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min3_i16 v1, v1, v5, v13
+; GFX9-SDAG-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX9-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v0, sext(v0), sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min3_i16 v1, v1, v3, v7
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v2, sext(v2), sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_min3_i16 v0, v0, v4, v12
+; GFX9-SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_min3_i16 v0, v0, v2, v6
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_min_i16_sdwa v0, v0, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_smin_v16i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v5, sext(v5), sext(v13) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v7, sext(v7), sext(v15) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v8, v8, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX9-GISEL-NEXT: v_bfe_i32 v10, v10, 0, 8
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v4, sext(v4), sext(v12) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_min_i16_sdwa v6, sext(v6), sext(v14) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_min3_i16 v1, v1, v9, v5
+; GFX9-GISEL-NEXT: v_min3_i16 v3, v3, v11, v7
+; GFX9-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX9-GISEL-NEXT: v_min3_i16 v0, v0, v8, v4
+; GFX9-GISEL-NEXT: v_min3_i16 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_min_i16_e32 v1, s0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_min3_i16 v2, v2, s0, s0
+; GFX9-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_smin_v16i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v1, v1, v9
+; GFX10-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v7, v7, v15
+; GFX10-SDAG-NEXT: v_min_i16 v3, v3, v11
+; GFX10-SDAG-NEXT: v_min3_i16 v1, v1, v5, v13
+; GFX10-SDAG-NEXT: v_bfe_i32 v5, v8, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v8, v10, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-SDAG-NEXT: v_min3_i16 v1, v1, v3, v7
+; GFX10-SDAG-NEXT: v_bfe_i32 v3, v14, 0, 8
+; GFX10-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v0, v0, v5
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_min_i16 v3, v6, v3
+; GFX10-SDAG-NEXT: v_min_i16 v2, v2, v8
+; GFX10-SDAG-NEXT: v_min3_i16 v0, v0, v4, v9
+; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX10-SDAG-NEXT: v_min3_i16 v0, v0, v2, v3
+; GFX10-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-SDAG-NEXT: v_min_i16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_smin_v16i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX10-GISEL-NEXT: v_min_i16 v5, v5, v13
+; GFX10-GISEL-NEXT: v_min_i16 v7, v7, v15
+; GFX10-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v13, v14, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v12, v12, 0, 8
+; GFX10-GISEL-NEXT: v_min3_i16 v1, v1, v9, v5
+; GFX10-GISEL-NEXT: v_min3_i16 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_bfe_i32 v5, v10, 0, 8
+; GFX10-GISEL-NEXT: v_min_i16 v6, v6, v13
+; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX10-GISEL-NEXT: v_bfe_i32 v7, v8, 0, 8
+; GFX10-GISEL-NEXT: v_min_i16 v4, v4, v12
+; GFX10-GISEL-NEXT: v_min_i16 v1, v1, v3
+; GFX10-GISEL-NEXT: v_min3_i16 v2, v2, v5, v6
+; GFX10-GISEL-NEXT: s_sext_i32_i8 s4, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-GISEL-NEXT: v_min3_i16 v0, v0, v7, v4
+; GFX10-GISEL-NEXT: v_min_i16 v4, v1, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v6, v2, s4, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v3, v3, s4, s4
+; GFX10-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -939,40 +1352,49 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_smin_v16i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX11-GISEL-NEXT: v_min_i16 v1, v1, v9
-; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX11-GISEL-NEXT: v_min_i16 v5, v5, v13
; GFX11-GISEL-NEXT: v_min_i16 v7, v7, v15
-; GFX11-GISEL-NEXT: v_min_i16 v3, v3, v11
-; GFX11-GISEL-NEXT: v_min3_i16 v1, v1, v5, v13
-; GFX11-GISEL-NEXT: v_bfe_i32 v5, v8, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX11-GISEL-NEXT: v_bfe_i32 v8, v10, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX11-GISEL-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX11-GISEL-NEXT: v_bfe_i32 v3, v14, 0, 8
; GFX11-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX11-GISEL-NEXT: v_min_i16 v0, v0, v5
+; GFX11-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX11-GISEL-NEXT: v_min3_i16 v1, v1, v9, v5
+; GFX11-GISEL-NEXT: v_min3_i16 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: v_bfe_i32 v5, v14, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX11-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX11-GISEL-NEXT: v_min_i16 v1, v1, v3
+; GFX11-GISEL-NEXT: v_min_i16 v5, v6, v5
+; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-GISEL-NEXT: v_bfe_i32 v6, v8, 0, 8
+; GFX11-GISEL-NEXT: v_min_i16 v4, v4, v9
+; GFX11-GISEL-NEXT: v_min_i16 v8, v1, s0
+; GFX11-GISEL-NEXT: v_min3_i16 v2, v2, v7, v5
+; GFX11-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v6, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v8
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: v_min_i16 v3, v6, v3
+; GFX11-GISEL-NEXT: v_min3_i16 v5, v2, s0, s0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_min_i16 v2, v2, v8
-; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v4, v9
+; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v4
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: v_min3_i16 v0, v0, v2, v3
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX11-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_smin_v16i8:
@@ -1089,40 +1511,50 @@ define i8 @test_vector_reduce_smin_v16i8(<16 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v5, v5, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v13, v13, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v7, v7, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v15, v15, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v9, v9, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v3, v3, 0, 8
-; GFX12-GISEL-NEXT: v_min_i16 v1, v1, v9
-; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v11, v11, 0, 8
+; GFX12-GISEL-NEXT: v_min_i16 v5, v5, v13
; GFX12-GISEL-NEXT: v_min_i16 v7, v7, v15
-; GFX12-GISEL-NEXT: v_min_i16 v3, v3, v11
-; GFX12-GISEL-NEXT: v_min3_i16 v1, v1, v5, v13
-; GFX12-GISEL-NEXT: v_bfe_i32 v5, v8, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
-; GFX12-GISEL-NEXT: v_bfe_i32 v8, v10, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v6, v6, 0, 8
-; GFX12-GISEL-NEXT: v_min3_i16 v1, v1, v3, v7
-; GFX12-GISEL-NEXT: v_bfe_i32 v3, v14, 0, 8
; GFX12-GISEL-NEXT: v_bfe_i32 v2, v2, 0, 8
-; GFX12-GISEL-NEXT: v_min_i16 v0, v0, v5
+; GFX12-GISEL-NEXT: v_bfe_i32 v4, v4, 0, 8
+; GFX12-GISEL-NEXT: v_min3_i16 v1, v1, v9, v5
+; GFX12-GISEL-NEXT: v_min3_i16 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: v_bfe_i32 v5, v14, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v7, v10, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v9, v12, 0, 8
+; GFX12-GISEL-NEXT: s_sext_i32_i8 s0, s0
+; GFX12-GISEL-NEXT: v_min_i16 v1, v1, v3
+; GFX12-GISEL-NEXT: v_min_i16 v5, v6, v5
+; GFX12-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-GISEL-NEXT: v_bfe_i32 v6, v8, 0, 8
+; GFX12-GISEL-NEXT: v_min_i16 v4, v4, v9
+; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-GISEL-NEXT: v_min_i16 v8, v1, s0
+; GFX12-GISEL-NEXT: v_min3_i16 v2, v2, v7, v5
+; GFX12-GISEL-NEXT: v_min3_i16 v3, v3, s0, s0
+; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v6, v4
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: v_min_i16 v3, v6, v3
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v8
+; GFX12-GISEL-NEXT: v_min3_i16 v5, v2, s0, s0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_min_i16 v2, v2, v8
-; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v4, v9
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v2, v3
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-GISEL-NEXT: v_min_i16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: v_min3_i16 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %v)
@@ -1131,19 +1563,41 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smin_v2i16(<2 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v2i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v2i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v2i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, v1, v0
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smin_v2i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1372,24 +1826,51 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smin_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v2, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v3, 16, v1
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-NEXT: v_min3_i32 v0, v0, v1, v2
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v3, 16, v1
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v1, v2
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_min3_i32 v1, v2, v3, v0
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smin_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1494,35 +1975,73 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_smin_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v5, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v7, 16, v2
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v4, 16, v1
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v6, 16, v3
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX7-NEXT: v_min_i32_e32 v5, v5, v7
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v2
-; GFX7-NEXT: v_min3_i32 v2, v5, v4, v6
-; GFX7-NEXT: v_min3_i32 v0, v0, v1, v3
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v5, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v7, 16, v2
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v4, 16, v1
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v6, 16, v3
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX7-SDAG-NEXT: v_min_i32_e32 v5, v5, v7
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_min3_i32 v2, v5, v4, v6
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v1, v3
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_e32 v1, v1, v3
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_min_i16_e32 v2, v5, v4
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v7, v3, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v2, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX7-GISEL-NEXT: v_min_i32_e32 v6, v6, v7
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_min3_i32 v3, v4, v5, v6
+; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_min_i32_e32 v1, v3, v0
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_min_i16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_i16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smin_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1652,57 +2171,116 @@ entry:
}
define i16 @test_vector_reduce_smin_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_smin_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_ashrrev_i32_e32 v9, 16, v0
-; GFX7-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v13, 16, v4
-; GFX7-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v8, 16, v2
-; GFX7-NEXT: v_ashrrev_i32_e32 v10, 16, v1
-; GFX7-NEXT: v_ashrrev_i32_e32 v11, 16, v3
-; GFX7-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX7-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX7-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX7-NEXT: v_ashrrev_i32_e32 v12, 16, v6
-; GFX7-NEXT: v_ashrrev_i32_e32 v14, 16, v5
-; GFX7-NEXT: v_ashrrev_i32_e32 v15, 16, v7
-; GFX7-NEXT: v_bfe_i32 v6, v6, 0, 16
-; GFX7-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX7-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX7-NEXT: v_min_i32_e32 v9, v9, v13
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v4
-; GFX7-NEXT: v_min_i32_e32 v11, v11, v15
-; GFX7-NEXT: v_min_i32_e32 v10, v10, v14
-; GFX7-NEXT: v_min_i32_e32 v3, v3, v7
-; GFX7-NEXT: v_min_i32_e32 v1, v1, v5
-; GFX7-NEXT: v_min3_i32 v4, v9, v8, v12
-; GFX7-NEXT: v_min3_i32 v0, v0, v2, v6
-; GFX7-NEXT: v_min3_i32 v2, v4, v10, v11
-; GFX7-NEXT: v_min3_i32 v0, v0, v1, v3
-; GFX7-NEXT: v_min_i32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_smin_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v9, 16, v0
+; GFX7-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v13, 16, v4
+; GFX7-SDAG-NEXT: v_bfe_i32 v4, v4, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v8, 16, v2
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v10, 16, v1
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v11, 16, v3
+; GFX7-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v12, 16, v6
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v14, 16, v5
+; GFX7-SDAG-NEXT: v_ashrrev_i32_e32 v15, 16, v7
+; GFX7-SDAG-NEXT: v_bfe_i32 v6, v6, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v5, v5, 0, 16
+; GFX7-SDAG-NEXT: v_bfe_i32 v7, v7, 0, 16
+; GFX7-SDAG-NEXT: v_min_i32_e32 v9, v9, v13
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_min_i32_e32 v11, v11, v15
+; GFX7-SDAG-NEXT: v_min_i32_e32 v10, v10, v14
+; GFX7-SDAG-NEXT: v_min_i32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_min_i32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_min3_i32 v4, v9, v8, v12
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_min3_i32 v2, v4, v10, v11
+; GFX7-SDAG-NEXT: v_min3_i32 v0, v0, v1, v3
+; GFX7-SDAG-NEXT: v_min_i32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_smin_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_i16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_i16_e32 v2, v2, v6
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v4
-; GFX8-NEXT: v_min_i16_e32 v3, v3, v7
-; GFX8-NEXT: v_min_i16_e32 v1, v1, v5
-; GFX8-NEXT: v_min_i16_e32 v4, v11, v10
-; GFX8-NEXT: v_min_i16_e32 v5, v9, v8
-; GFX8-NEXT: v_min_i16_e32 v1, v1, v3
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_min_i16_e32 v2, v5, v4
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_i16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_smin_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_bfe_i32 v12, v2, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v13, v6, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v6, 16, 16
+; GFX7-GISEL-NEXT: v_min_i32_e32 v12, v12, v13
+; GFX7-GISEL-NEXT: v_min_i32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_bfe_i32 v6, v3, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v13, v7, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v3, v3, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v7, v7, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v8, v0, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v9, v4, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v4, v4, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v10, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v11, v5, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX7-GISEL-NEXT: v_bfe_i32 v5, v5, 16, 16
+; GFX7-GISEL-NEXT: v_min_i32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_min_i32_e32 v6, v6, v13
+; GFX7-GISEL-NEXT: v_min3_i32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_min3_i32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_min3_i32 v7, v8, v9, v12
+; GFX7-GISEL-NEXT: v_min3_i32 v2, v10, v11, v6
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_min3_i32 v1, v7, v2, v0
+; GFX7-GISEL-NEXT: v_min_i32_e32 v0, 0, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_smin_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_i16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_min_i16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_min_i16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_min_i16_e32 v4, v11, v10
+; GFX8-SDAG-NEXT: v_min_i16_e32 v5, v9, v8
+; GFX8-SDAG-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_min_i16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_smin_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_i16_e32 v8, v0, v4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v4, v1, v5
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v5, v2, v6
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v6, v3, v7
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, v4, v6
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_min_i16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_min_i16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_i16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_smin_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
index 0781caaaf61f8..7e7da135b05f3 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umax.ll
@@ -1,21 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_umax_v2i8(<2 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_umax_v2i8:
@@ -278,49 +278,136 @@ entry:
}
define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v4i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_max3_u32 v0, v0, v2, v1
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v4i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v2, v1
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v4i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v4i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v2, v2, 0, 0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v3, v3, 0, 0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_umax_v4i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX9-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_max3_u16 v0, v0, v2, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v4i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_umax_v4i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: v_max_u16 v1, v1, v3
-; GFX10-NEXT: v_mov_b32_e32 v3, 8
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshrrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_max3_u16 v0, v0, v2, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v4i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, 0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, 0, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v3, 0, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_umax_v4i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_umax_v4i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_max_u16_e32 v1, 0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX9-GISEL-NEXT: v_max3_u16 v2, v2, 0, 0
+; GFX9-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_umax_v4i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: v_max_u16 v1, v1, v3
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, 8
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_lshrrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-SDAG-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_umax_v4i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-GISEL-NEXT: v_max_u16 v1, v1, v3
+; GFX10-GISEL-NEXT: v_max3_u16 v5, v2, 0, 0
+; GFX10-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX10-GISEL-NEXT: v_max_u16 v6, v1, 0
+; GFX10-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v4, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -359,18 +446,28 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_umax_v4i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-GISEL-NEXT: v_max_u16 v1, v1, v3
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_max3_u16 v5, v2, 0, 0
+; GFX11-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_max_u16 v4, v1, 0
; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v4i8:
@@ -422,18 +519,28 @@ define i8 @test_vector_reduce_umax_v4i8(<4 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_max_u16 v1, v1, v3
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_max3_u16 v5, v2, 0, 0
+; GFX12-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_max_u16 v4, v1, 0
; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.umax.v4i8(<4 x i8> %v)
@@ -441,73 +548,181 @@ entry:
}
define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v8i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v4
-; GFX7-NEXT: v_max_u32_e32 v1, v1, v5
-; GFX7-NEXT: v_max3_u32 v1, v1, v3, v7
-; GFX7-NEXT: v_max3_u32 v0, v0, v2, v6
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v8i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_max_u32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_max3_u32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v8i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v8i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-GISEL-NEXT: v_max_u32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_max_u32_e32 v3, v3, v6
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_max3_u32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v2, v2, 0, 0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v3, v3, 0, 0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_umax_v8i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX9-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX9-NEXT: v_max_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX9-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX9-NEXT: v_max_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_max3_u16 v0, v0, v2, v6
-; GFX9-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX9-NEXT: v_max_u16_e32 v0, v0, v1
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v8i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_umax_v8i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: v_max_u16 v1, v1, v5
-; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX10-NEXT: v_max_u16 v0, v0, v4
-; GFX10-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v6
-; GFX10-NEXT: v_mov_b32_e32 v4, 8
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_max3_u16 v0, v0, v2, v3
-; GFX10-NEXT: v_lshrrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_max_u16 v0, v0, v1
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v8i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, 0, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, 0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, 0, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_e32 v3, 0, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v3, 0, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_umax_v8i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_u16 v1, v1, v3, v7
+; GFX9-SDAG-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX9-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max3_u16 v0, v0, v2, v6
+; GFX9-SDAG-NEXT: v_lshrrev_b32_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_umax_v8i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max3_u16 v0, v0, v4, v2
+; GFX9-GISEL-NEXT: v_max3_u16 v1, v1, v5, v3
+; GFX9-GISEL-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX9-GISEL-NEXT: v_max_u16_e32 v1, 0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v8, 0xff
+; GFX9-GISEL-NEXT: v_max3_u16 v2, v2, 0, 0
+; GFX9-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v8, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_umax_v8i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: v_max_u16 v1, v1, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-SDAG-NEXT: v_max_u16 v0, v0, v4
+; GFX10-SDAG-NEXT: v_max3_u16 v1, v1, v3, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v6
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, 8
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_max3_u16 v0, v0, v2, v3
+; GFX10-SDAG-NEXT: v_lshrrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-SDAG-NEXT: v_max_u16 v0, v0, v1
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_umax_v8i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX10-GISEL-NEXT: v_max_u16 v3, v3, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX10-GISEL-NEXT: v_max_u16 v2, v2, v6
+; GFX10-GISEL-NEXT: v_max3_u16 v1, v1, v5, v3
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX10-GISEL-NEXT: v_max3_u16 v0, v0, v4, v2
+; GFX10-GISEL-NEXT: v_max_u16 v4, v1, 0
+; GFX10-GISEL-NEXT: v_max3_u16 v2, v2, 0, 0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -562,26 +777,36 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_umax_v8i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-GISEL-NEXT: v_max_u16 v1, v1, v5
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-GISEL-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v4
-; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v6
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_max_u16 v0, v0, v3
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v2, v4
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX11-GISEL-NEXT: v_max_u16 v3, v3, v7
+; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_max_u16 v2, v2, v6
+; GFX11-GISEL-NEXT: v_max3_u16 v1, v1, v5, v3
+; GFX11-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v4, v2
+; GFX11-GISEL-NEXT: v_max_u16 v5, v1, 0
+; GFX11-GISEL-NEXT: v_max3_u16 v2, v2, 0, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX11-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v8i8:
@@ -649,26 +874,36 @@ define i8 @test_vector_reduce_umax_v8i8(<8 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX12-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX12-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX12-GISEL-NEXT: v_max_u16 v1, v1, v5
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX12-GISEL-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v4
-; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v6
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_max_u16 v0, v0, v3
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v2, v4
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX12-GISEL-NEXT: v_max_u16 v3, v3, v7
+; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_max_u16 v2, v2, v6
+; GFX12-GISEL-NEXT: v_max3_u16 v1, v1, v5, v3
+; GFX12-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v4, v2
+; GFX12-GISEL-NEXT: v_max_u16 v5, v1, 0
+; GFX12-GISEL-NEXT: v_max3_u16 v2, v2, 0, 0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
; GFX12-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> %v)
@@ -676,121 +911,272 @@ entry:
}
define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v16i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_and_b32_e32 v13, 0xff, v13
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_and_b32_e32 v12, 0xff, v12
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX7-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_and_b32_e32 v15, 0xff, v15
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_and_b32_e32 v10, 0xff, v10
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_max_u32_e32 v1, v1, v9
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v8
-; GFX7-NEXT: v_max_u32_e32 v6, v6, v14
-; GFX7-NEXT: v_max_u32_e32 v2, v2, v10
-; GFX7-NEXT: v_max_u32_e32 v7, v7, v15
-; GFX7-NEXT: v_max_u32_e32 v3, v3, v11
-; GFX7-NEXT: v_max3_u32 v0, v0, v4, v12
-; GFX7-NEXT: v_max3_u32 v1, v1, v5, v13
-; GFX7-NEXT: v_max3_u32 v1, v1, v3, v7
-; GFX7-NEXT: v_max3_u32 v0, v0, v2, v6
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v16i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v14, 0xff, v14
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-SDAG-NEXT: v_max_u32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_max_u32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_max_u32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_max_u32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_max_u32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v4, v12
+; GFX7-SDAG-NEXT: v_max3_u32 v1, v1, v5, v13
+; GFX7-SDAG-NEXT: v_max3_u32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v16i8:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_max_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_max_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_max_u16_e32 v2, v2, v6
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, 8
-; GFX8-NEXT: v_lshrrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX8-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v16i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX7-GISEL-NEXT: v_max_u32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v13
+; GFX7-GISEL-NEXT: v_max_u32_e32 v5, v5, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v14
+; GFX7-GISEL-NEXT: v_max_u32_e32 v6, v6, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX7-GISEL-NEXT: v_max_u32_e32 v7, v7, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX7-GISEL-NEXT: v_max3_u32 v1, v1, v9, v5
+; GFX7-GISEL-NEXT: v_max3_u32 v3, v3, v11, v7
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v8, v4
+; GFX7-GISEL-NEXT: v_max3_u32 v2, v2, v10, v6
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v2, v2, 0, 0
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v3, v3, 0, 0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX9-LABEL: test_vector_reduce_umax_v16i8:
-; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v13, 0xff, v13
-; GFX9-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX9-NEXT: v_max_u16_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_and_b32_e32 v12, 0xff, v12
-; GFX9-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX9-NEXT: v_max_u16_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max_u16_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max_u16_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_u16 v1, v1, v5, v13
-; GFX9-NEXT: v_max_u16_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max_u16_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_max3_u16 v0, v0, v4, v12
-; GFX9-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX9-NEXT: v_max3_u16 v0, v0, v2, v6
-; GFX9-NEXT: v_lshlrev_b16_e32 v1, 8, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, 8
-; GFX9-NEXT: v_lshrrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX9-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v16i8:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-SDAG-NEXT: v_max_u16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_max_u16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_max_u16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, 8
+; GFX8-SDAG-NEXT: v_lshrrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10-LABEL: test_vector_reduce_umax_v16i8:
-; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX10-NEXT: v_and_b32_e32 v13, 0xff, v13
-; GFX10-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX10-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX10-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX10-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX10-NEXT: v_and_b32_e32 v15, 0xff, v15
-; GFX10-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX10-NEXT: v_max_u16 v1, v1, v9
-; GFX10-NEXT: v_and_b32_e32 v12, 0xff, v12
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX10-NEXT: v_and_b32_e32 v10, 0xff, v10
-; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX10-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX10-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX10-NEXT: v_max_u16 v7, v7, v15
-; GFX10-NEXT: v_max_u16 v3, v3, v11
-; GFX10-NEXT: v_max_u16 v0, v0, v8
-; GFX10-NEXT: v_max3_u16 v1, v1, v5, v13
-; GFX10-NEXT: v_max_u16 v5, v6, v14
-; GFX10-NEXT: v_max_u16 v2, v2, v10
-; GFX10-NEXT: v_max3_u16 v0, v0, v4, v12
-; GFX10-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX10-NEXT: v_max3_u16 v0, v0, v2, v5
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX10-NEXT: v_mov_b32_e32 v1, 8
-; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v0
-; GFX10-NEXT: v_lshrrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
-; GFX10-NEXT: v_max_u16 v0, v2, v0
-; GFX10-NEXT: s_setpc_b64 s[30:31]
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v16i8:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v4
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v1, v5
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_max_u16_e32 v3, v3, v7
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, 0, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, 0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, 0, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-GISEL-NEXT: v_max_u16_e32 v3, 0, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v3, 0, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-SDAG-LABEL: test_vector_reduce_umax_v16i8:
+; GFX9-SDAG: ; %bb.0: ; %entry
+; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX9-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v1, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX9-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v3, v3, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_u16 v1, v1, v5, v13
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v2, v2, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-SDAG-NEXT: v_max3_u16 v0, v0, v4, v12
+; GFX9-SDAG-NEXT: v_max3_u16 v1, v1, v3, v7
+; GFX9-SDAG-NEXT: v_max3_u16 v0, v0, v2, v6
+; GFX9-SDAG-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 8
+; GFX9-SDAG-NEXT: v_lshrrev_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX9-SDAG-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-GISEL-LABEL: test_vector_reduce_umax_v16i8:
+; GFX9-GISEL: ; %bb.0: ; %entry
+; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX9-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX9-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v5, v5, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v7, v7, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX9-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v4, v4, v12 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max_u16_sdwa v6, v6, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_max3_u16 v1, v1, v9, v5
+; GFX9-GISEL-NEXT: v_max3_u16 v3, v3, v11, v7
+; GFX9-GISEL-NEXT: v_max3_u16 v0, v0, v8, v4
+; GFX9-GISEL-NEXT: v_max3_u16 v2, v2, v10, v6
+; GFX9-GISEL-NEXT: v_max_u16_e32 v1, v1, v3
+; GFX9-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX9-GISEL-NEXT: v_max_u16_e32 v1, 0, v1
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 8
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v16, 0xff
+; GFX9-GISEL-NEXT: v_max3_u16 v2, v2, 0, 0
+; GFX9-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX9-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-GISEL-NEXT: v_and_or_b32 v0, v0, v16, v1
+; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX9-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-SDAG-LABEL: test_vector_reduce_umax_v16i8:
+; GFX10-SDAG: ; %bb.0: ; %entry
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-SDAG-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-SDAG-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX10-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX10-SDAG-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-SDAG-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX10-SDAG-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX10-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX10-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-SDAG-NEXT: v_max_u16 v1, v1, v9
+; GFX10-SDAG-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX10-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX10-SDAG-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-SDAG-NEXT: v_and_b32_e32 v14, 0xff, v14
+; GFX10-SDAG-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX10-SDAG-NEXT: v_max_u16 v7, v7, v15
+; GFX10-SDAG-NEXT: v_max_u16 v3, v3, v11
+; GFX10-SDAG-NEXT: v_max_u16 v0, v0, v8
+; GFX10-SDAG-NEXT: v_max3_u16 v1, v1, v5, v13
+; GFX10-SDAG-NEXT: v_max_u16 v5, v6, v14
+; GFX10-SDAG-NEXT: v_max_u16 v2, v2, v10
+; GFX10-SDAG-NEXT: v_max3_u16 v0, v0, v4, v12
+; GFX10-SDAG-NEXT: v_max3_u16 v1, v1, v3, v7
+; GFX10-SDAG-NEXT: v_max3_u16 v0, v0, v2, v5
+; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v1
+; GFX10-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX10-SDAG-NEXT: v_lshrrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX10-SDAG-NEXT: v_max_u16 v0, v2, v0
+; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-GISEL-LABEL: test_vector_reduce_umax_v16i8:
+; GFX10-GISEL: ; %bb.0: ; %entry
+; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX10-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX10-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX10-GISEL-NEXT: v_max_u16 v5, v5, v13
+; GFX10-GISEL-NEXT: v_max_u16 v7, v7, v15
+; GFX10-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX10-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v14
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX10-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX10-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX10-GISEL-NEXT: v_max3_u16 v1, v1, v9, v5
+; GFX10-GISEL-NEXT: v_max3_u16 v3, v3, v11, v7
+; GFX10-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v10
+; GFX10-GISEL-NEXT: v_max_u16 v6, v6, v13
+; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v8
+; GFX10-GISEL-NEXT: v_max_u16 v4, v4, v12
+; GFX10-GISEL-NEXT: v_max_u16 v1, v1, v3
+; GFX10-GISEL-NEXT: v_max3_u16 v2, v2, v5, v6
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GFX10-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX10-GISEL-NEXT: v_max3_u16 v0, v0, v7, v4
+; GFX10-GISEL-NEXT: v_max_u16 v4, v1, 0
+; GFX10-GISEL-NEXT: v_max3_u16 v6, v2, 0, 0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX10-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX10-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX10-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
@@ -877,42 +1263,48 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX11-GISEL-LABEL: test_vector_reduce_umax_v16i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX11-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX11-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
; GFX11-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX11-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-GISEL-NEXT: v_max_u16 v1, v1, v9
-; GFX11-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
-; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX11-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
-; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX11-GISEL-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX11-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX11-GISEL-NEXT: v_max_u16 v5, v5, v13
; GFX11-GISEL-NEXT: v_max_u16 v7, v7, v15
-; GFX11-GISEL-NEXT: v_max_u16 v3, v3, v11
-; GFX11-GISEL-NEXT: v_max_u16 v0, v0, v8
-; GFX11-GISEL-NEXT: v_max3_u16 v1, v1, v5, v13
-; GFX11-GISEL-NEXT: v_max_u16 v5, v6, v14
-; GFX11-GISEL-NEXT: v_max_u16 v2, v2, v10
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v4, v12
-; GFX11-GISEL-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v2, v5
-; GFX11-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX11-GISEL-NEXT: v_max3_u16 v1, v1, v9, v5
+; GFX11-GISEL-NEXT: v_max3_u16 v3, v3, v11, v7
+; GFX11-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v14
+; GFX11-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v10
+; GFX11-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v12
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX11-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX11-GISEL-NEXT: v_max_u16 v1, v1, v3
+; GFX11-GISEL-NEXT: v_max_u16 v5, v6, v5
+; GFX11-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v8
+; GFX11-GISEL-NEXT: v_max_u16 v4, v4, v9
+; GFX11-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX11-GISEL-NEXT: v_max_u16 v8, v1, 0
+; GFX11-GISEL-NEXT: v_max3_u16 v2, v2, v7, v5
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v6, v4
+; GFX11-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v8
+; GFX11-GISEL-NEXT: v_max3_u16 v5, v2, 0, 0
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX11-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_umax_v16i8:
@@ -1012,42 +1404,48 @@ define i8 @test_vector_reduce_umax_v16i8(<16 x i8> %v) {
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX12-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
; GFX12-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX12-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX12-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX12-GISEL-NEXT: v_and_b32_e32 v13, 0xff, v13
; GFX12-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX12-GISEL-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX12-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX12-GISEL-NEXT: v_max_u16 v1, v1, v9
-; GFX12-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
-; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX12-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
-; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX12-GISEL-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX12-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX12-GISEL-NEXT: v_max_u16 v5, v5, v13
; GFX12-GISEL-NEXT: v_max_u16 v7, v7, v15
-; GFX12-GISEL-NEXT: v_max_u16 v3, v3, v11
-; GFX12-GISEL-NEXT: v_max_u16 v0, v0, v8
-; GFX12-GISEL-NEXT: v_max3_u16 v1, v1, v5, v13
-; GFX12-GISEL-NEXT: v_max_u16 v5, v6, v14
-; GFX12-GISEL-NEXT: v_max_u16 v2, v2, v10
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v4, v12
-; GFX12-GISEL-NEXT: v_max3_u16 v1, v1, v3, v7
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v2, v5
-; GFX12-GISEL-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX12-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX12-GISEL-NEXT: v_max3_u16 v1, v1, v9, v5
+; GFX12-GISEL-NEXT: v_max3_u16 v3, v3, v11, v7
+; GFX12-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v14
+; GFX12-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v10
+; GFX12-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v12
; GFX12-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-GISEL-NEXT: v_max_u16 v0, v0, v1
+; GFX12-GISEL-NEXT: v_max_u16 v1, v1, v3
+; GFX12-GISEL-NEXT: v_max_u16 v5, v6, v5
+; GFX12-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v8
+; GFX12-GISEL-NEXT: v_max_u16 v4, v4, v9
+; GFX12-GISEL-NEXT: v_max3_u16 v3, v3, 0, 0
+; GFX12-GISEL-NEXT: v_max_u16 v8, v1, 0
+; GFX12-GISEL-NEXT: v_max3_u16 v2, v2, v7, v5
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v6, v4
+; GFX12-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v8
+; GFX12-GISEL-NEXT: v_max3_u16 v5, v2, 0, 0
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-GISEL-NEXT: v_max3_u16 v0, v0, v2, v1
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v5
+; GFX12-GISEL-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX12-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX12-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %v)
@@ -1056,19 +1454,39 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umax_v2i16(<2 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v2i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v2i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v2i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umax_v2i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1290,24 +1708,49 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umax_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_max3_u32 v0, v0, v1, v2
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v1, v2
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_max_u32_e32 v2, v2, v3
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v1, v2
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umax_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1412,35 +1855,71 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umax_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v2
-; GFX7-NEXT: v_max_u32_e32 v2, v5, v7
-; GFX7-NEXT: v_max3_u32 v1, v2, v1, v3
-; GFX7-NEXT: v_max3_u32 v0, v0, v4, v6
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v3
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_max_u32_e32 v2, v5, v7
+; GFX7-SDAG-NEXT: v_max3_u32 v1, v2, v1, v3
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v4, v6
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_max_u32_e32 v3, v5, v7
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_max3_u32 v1, v4, v6, v3
+; GFX7-GISEL-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_u16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umax_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1571,57 +2050,114 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umax_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umax_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v13, 0xffff, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v6
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v7
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_max_u32_e32 v0, v0, v4
-; GFX7-NEXT: v_max_u32_e32 v4, v9, v13
-; GFX7-NEXT: v_max_u32_e32 v3, v3, v7
-; GFX7-NEXT: v_max_u32_e32 v1, v1, v5
-; GFX7-NEXT: v_max_u32_e32 v5, v11, v15
-; GFX7-NEXT: v_max_u32_e32 v7, v10, v14
-; GFX7-NEXT: v_max3_u32 v2, v4, v2, v6
-; GFX7-NEXT: v_max3_u32 v0, v0, v8, v12
-; GFX7-NEXT: v_max3_u32 v0, v0, v7, v5
-; GFX7-NEXT: v_max3_u32 v1, v2, v1, v3
-; GFX7-NEXT: v_max_u32_e32 v0, v1, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umax_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v9, 0xffff, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v13, 0xffff, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v5
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_max_u32_e32 v4, v9, v13
+; GFX7-SDAG-NEXT: v_max_u32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_max_u32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_max_u32_e32 v5, v11, v15
+; GFX7-SDAG-NEXT: v_max_u32_e32 v7, v10, v14
+; GFX7-SDAG-NEXT: v_max3_u32 v2, v4, v2, v6
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v8, v12
+; GFX7-SDAG-NEXT: v_max3_u32 v0, v0, v7, v5
+; GFX7-SDAG-NEXT: v_max3_u32 v1, v2, v1, v3
+; GFX7-SDAG-NEXT: v_max_u32_e32 v0, v1, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umax_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_max_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_u16_e32 v2, v2, v6
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_max_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_max_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_max_u16_e32 v4, v11, v10
-; GFX8-NEXT: v_max_u16_e32 v5, v9, v8
-; GFX8-NEXT: v_max_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_max_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_max_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umax_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-GISEL-NEXT: v_max_u32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_max_u32_e32 v6, v10, v14
+; GFX7-GISEL-NEXT: v_max_u32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_max_u32_e32 v7, v11, v15
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_max3_u32 v2, v8, v12, v6
+; GFX7-GISEL-NEXT: v_max3_u32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_max3_u32 v3, v9, v13, v7
+; GFX7-GISEL-NEXT: v_max_u32_e32 v2, v2, v3
+; GFX7-GISEL-NEXT: v_max3_u32 v0, v0, v1, v2
+; GFX7-GISEL-NEXT: v_max_u32_e32 v1, 0, v2
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umax_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_max_u16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_max_u16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_max_u16_e32 v4, v11, v10
+; GFX8-SDAG-NEXT: v_max_u16_e32 v5, v9, v8
+; GFX8-SDAG-NEXT: v_max_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_max_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umax_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_u16_e32 v8, v0, v4
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v4, v1, v5
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v5, v2, v6
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v6, v3, v7
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, v4, v6
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_max_u16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_max_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_u16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umax_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index e7e4cc790e026..5527a543f9f5f 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
@@ -278,16 +278,27 @@ entry:
}
define i8 @test_vector_reduce_umin_v4i8(<4 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_umin_v4i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_min3_u32 v0, v0, v2, v1
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umin_v4i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v2, v1
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_umin_v4i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-GISEL-NEXT: v_min_u32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_umin_v4i8:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -466,23 +477,41 @@ entry:
}
define i8 @test_vector_reduce_umin_v8i8(<8 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_umin_v8i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v4
-; GFX7-NEXT: v_min_u32_e32 v1, v1, v5
-; GFX7-NEXT: v_min3_u32 v1, v1, v3, v7
-; GFX7-NEXT: v_min3_u32 v0, v0, v2, v6
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umin_v8i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_min_u32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_min3_u32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_umin_v8i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-GISEL-NEXT: v_min_u32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_min_u32_e32 v3, v3, v6
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_min3_u32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_umin_v8i8:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -740,37 +769,68 @@ entry:
}
define i8 @test_vector_reduce_umin_v16i8(<16 x i8> %v) {
-; GFX7-LABEL: test_vector_reduce_umin_v16i8:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v8, 0xff, v8
-; GFX7-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX7-NEXT: v_and_b32_e32 v9, 0xff, v9
-; GFX7-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX7-NEXT: v_and_b32_e32 v13, 0xff, v13
-; GFX7-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX7-NEXT: v_and_b32_e32 v12, 0xff, v12
-; GFX7-NEXT: v_and_b32_e32 v4, 0xff, v4
-; GFX7-NEXT: v_and_b32_e32 v11, 0xff, v11
-; GFX7-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX7-NEXT: v_and_b32_e32 v15, 0xff, v15
-; GFX7-NEXT: v_and_b32_e32 v7, 0xff, v7
-; GFX7-NEXT: v_and_b32_e32 v10, 0xff, v10
-; GFX7-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX7-NEXT: v_and_b32_e32 v14, 0xff, v14
-; GFX7-NEXT: v_and_b32_e32 v6, 0xff, v6
-; GFX7-NEXT: v_min_u32_e32 v1, v1, v9
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v8
-; GFX7-NEXT: v_min_u32_e32 v6, v6, v14
-; GFX7-NEXT: v_min_u32_e32 v2, v2, v10
-; GFX7-NEXT: v_min_u32_e32 v7, v7, v15
-; GFX7-NEXT: v_min_u32_e32 v3, v3, v11
-; GFX7-NEXT: v_min3_u32 v0, v0, v4, v12
-; GFX7-NEXT: v_min3_u32 v1, v1, v5, v13
-; GFX7-NEXT: v_min3_u32 v1, v1, v3, v7
-; GFX7-NEXT: v_min3_u32 v0, v0, v2, v6
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umin_v16i8:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v13, 0xff, v13
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-SDAG-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v15, 0xff, v15
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v14, 0xff, v14
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-SDAG-NEXT: v_min_u32_e32 v1, v1, v9
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v8
+; GFX7-SDAG-NEXT: v_min_u32_e32 v6, v6, v14
+; GFX7-SDAG-NEXT: v_min_u32_e32 v2, v2, v10
+; GFX7-SDAG-NEXT: v_min_u32_e32 v7, v7, v15
+; GFX7-SDAG-NEXT: v_min_u32_e32 v3, v3, v11
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v4, v12
+; GFX7-SDAG-NEXT: v_min3_u32 v1, v1, v5, v13
+; GFX7-SDAG-NEXT: v_min3_u32 v1, v1, v3, v7
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v2, v6
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: test_vector_reduce_umin_v16i8:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v12
+; GFX7-GISEL-NEXT: v_min_u32_e32 v4, v4, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v13
+; GFX7-GISEL-NEXT: v_min_u32_e32 v5, v5, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xff, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v14
+; GFX7-GISEL-NEXT: v_min_u32_e32 v6, v6, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xff, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v12, 0xff, v15
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v9, 0xff, v9
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v11, 0xff, v11
+; GFX7-GISEL-NEXT: v_min_u32_e32 v7, v7, v12
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v8
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v10, 0xff, v10
+; GFX7-GISEL-NEXT: v_min3_u32 v1, v1, v9, v5
+; GFX7-GISEL-NEXT: v_min3_u32 v3, v3, v11, v7
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v8, v4
+; GFX7-GISEL-NEXT: v_min3_u32 v2, v2, v10, v6
+; GFX7-GISEL-NEXT: v_min_u32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_umin_v16i8:
; GFX8-SDAG: ; %bb.0: ; %entry
@@ -1193,11 +1253,20 @@ define i16 @test_vector_reduce_umin_v2i16(<2 x i16> %v) {
; GFX7-NEXT: v_min_u32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umin_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_vector_reduce_umin_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umin_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umin_v2i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1421,24 +1490,46 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umin_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umin_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_min3_u32 v0, v0, v1, v2
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umin_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v1, v2
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v3
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umin_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umin_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_min_u32_e32 v2, v2, v3
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v1, v2
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umin_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umin_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_u16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umin_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1543,35 +1634,68 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umin_v8i16(<8 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umin_v8i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v2
-; GFX7-NEXT: v_min_u32_e32 v2, v5, v7
-; GFX7-NEXT: v_min3_u32 v1, v2, v1, v3
-; GFX7-NEXT: v_min3_u32 v0, v0, v4, v6
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umin_v8i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v3
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v2
+; GFX7-SDAG-NEXT: v_min_u32_e32 v2, v5, v7
+; GFX7-SDAG-NEXT: v_min3_u32 v1, v2, v1, v3
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v4, v6
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umin_v8i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_min_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umin_v8i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_min_u32_e32 v1, v1, v3
+; GFX7-GISEL-NEXT: v_min_u32_e32 v3, v5, v7
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v2, v1
+; GFX7-GISEL-NEXT: v_min3_u32 v1, v4, v6, v3
+; GFX7-GISEL-NEXT: v_min_u32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umin_v8i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_min_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umin_v8i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_u16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_min_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umin_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1702,57 +1826,111 @@ entry:
; FIXME: With RegBankLegalize, v_alignbit_b32 is regression. Need pattern to look through COPY.
define i16 @test_vector_reduce_umin_v16i16(<16 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_umin_v16i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v9, 0xffff, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v13, 0xffff, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v12, 16, v6
-; GFX7-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; GFX7-NEXT: v_lshrrev_b32_e32 v15, 16, v7
-; GFX7-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_min_u32_e32 v0, v0, v4
-; GFX7-NEXT: v_min_u32_e32 v4, v9, v13
-; GFX7-NEXT: v_min_u32_e32 v3, v3, v7
-; GFX7-NEXT: v_min_u32_e32 v1, v1, v5
-; GFX7-NEXT: v_min_u32_e32 v5, v11, v15
-; GFX7-NEXT: v_min_u32_e32 v7, v10, v14
-; GFX7-NEXT: v_min3_u32 v2, v4, v2, v6
-; GFX7-NEXT: v_min3_u32 v0, v0, v8, v12
-; GFX7-NEXT: v_min3_u32 v0, v0, v7, v5
-; GFX7-NEXT: v_min3_u32 v1, v2, v1, v3
-; GFX7-NEXT: v_min_u32_e32 v0, v1, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_umin_v16i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v9, 0xffff, v0
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v13, 0xffff, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; GFX7-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v14, 16, v5
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v0, v4
+; GFX7-SDAG-NEXT: v_min_u32_e32 v4, v9, v13
+; GFX7-SDAG-NEXT: v_min_u32_e32 v3, v3, v7
+; GFX7-SDAG-NEXT: v_min_u32_e32 v1, v1, v5
+; GFX7-SDAG-NEXT: v_min_u32_e32 v5, v11, v15
+; GFX7-SDAG-NEXT: v_min_u32_e32 v7, v10, v14
+; GFX7-SDAG-NEXT: v_min3_u32 v2, v4, v2, v6
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v8, v12
+; GFX7-SDAG-NEXT: v_min3_u32 v0, v0, v7, v5
+; GFX7-SDAG-NEXT: v_min3_u32 v1, v2, v1, v3
+; GFX7-SDAG-NEXT: v_min_u32_e32 v0, v1, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_umin_v16i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_min_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_u16_e32 v2, v2, v6
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_min_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_min_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_min_u16_e32 v4, v11, v10
-; GFX8-NEXT: v_min_u16_e32 v5, v9, v8
-; GFX8-NEXT: v_min_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_min_u16_e32 v2, v5, v4
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_min_u16_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_umin_v16i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX7-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX7-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX7-GISEL-NEXT: v_min_u32_e32 v2, v2, v6
+; GFX7-GISEL-NEXT: v_min_u32_e32 v6, v10, v14
+; GFX7-GISEL-NEXT: v_min_u32_e32 v3, v3, v7
+; GFX7-GISEL-NEXT: v_min_u32_e32 v7, v11, v15
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v4, v2
+; GFX7-GISEL-NEXT: v_min3_u32 v2, v8, v12, v6
+; GFX7-GISEL-NEXT: v_min3_u32 v1, v1, v5, v3
+; GFX7-GISEL-NEXT: v_min3_u32 v3, v9, v13, v7
+; GFX7-GISEL-NEXT: v_min_u32_e32 v2, v2, v3
+; GFX7-GISEL-NEXT: v_min3_u32 v0, v0, v1, v2
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_umin_v16i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_u16_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_min_u16_e32 v3, v3, v7
+; GFX8-SDAG-NEXT: v_min_u16_e32 v1, v1, v5
+; GFX8-SDAG-NEXT: v_min_u16_e32 v4, v11, v10
+; GFX8-SDAG-NEXT: v_min_u16_e32 v5, v9, v8
+; GFX8-SDAG-NEXT: v_min_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_min_u16_e32 v2, v5, v4
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_min_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_umin_v16i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_min_u16_e32 v8, v0, v4
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v4, v1, v5
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v5, v2, v6
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v6, v3, v7
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v5, v8, v5
+; GFX8-GISEL-NEXT: v_min_u16_e32 v0, v0, v2
+; GFX8-GISEL-NEXT: v_min_u16_e32 v2, v4, v6
+; GFX8-GISEL-NEXT: v_min_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_min_u16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_min_u16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_min_u16_e32 v1, v2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-GISEL-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_umin_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
More information about the llvm-branch-commits
mailing list