[llvm-branch-commits] [llvm] AMDGPU/GlobalISel: Fix G_MERGE_VALUES lowering for extended LLTs (PR #209202)
Petar Avramovic via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Jul 20 03:42:02 PDT 2026
https://github.com/petar-avramovic updated https://github.com/llvm/llvm-project/pull/209202
>From 8e03e3aa291033ddff4f349272ffa15065e75f77 Mon Sep 17 00:00:00 2001
From: Petar Avramovic <Petar.Avramovic at amd.com>
Date: Thu, 16 Jul 2026 17:53:22 +0200
Subject: [PATCH] AMDGPU/GlobalISel: Fix G_MERGE_VALUES lowering for extended
LLTs
Use integer type for bit twiddling instead of scalar.
---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 2 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll | 41 +-
.../CodeGen/AMDGPU/GlobalISel/add.v2i16.ll | 295 +-
.../CodeGen/AMDGPU/GlobalISel/add.vni16.ll | 365 +--
llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll | 244 +-
...artifact-combiner-cse-leaves-dead-cast.mir | 60 +-
.../GlobalISel/artifact-combiner-sext.mir | 10 +-
.../artifact-combiner-unmerge-values.mir | 157 +-
.../GlobalISel/artifact-combiner-zext.mir | 8 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll | 424 +--
llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll | 54 +-
.../AMDGPU/GlobalISel/cvt_f32_ubyte.ll | 116 +-
.../CodeGen/AMDGPU/GlobalISel/dummy-target.ll | 16 +-
.../CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll | 1939 ++++++++----
llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll | 1063 ++-----
.../GlobalISel/fmed3-min-max-const-combine.ll | 15 +-
.../CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll | 373 ++-
llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll | 240 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll | 1629 ++++++----
llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll | 1691 ++++++----
.../AMDGPU/GlobalISel/legalize-abs.mir | 28 +-
.../AMDGPU/GlobalISel/legalize-add.mir | 44 +-
.../AMDGPU/GlobalISel/legalize-and.mir | 47 +-
.../AMDGPU/GlobalISel/legalize-anyext.mir | 26 +-
.../AMDGPU/GlobalISel/legalize-ashr.mir | 91 +-
.../AMDGPU/GlobalISel/legalize-bitcast.mir | 625 ++--
.../AMDGPU/GlobalISel/legalize-bitreverse.mir | 2 +-
.../AMDGPU/GlobalISel/legalize-bswap.mir | 39 +-
.../legalize-build-vector-trunc.mir | 10 +-
.../GlobalISel/legalize-build-vector.s16.mir | 208 +-
.../GlobalISel/legalize-ctlz-zero-poison.mir | 9 +-
.../AMDGPU/GlobalISel/legalize-ctlz.mir | 7 +-
.../AMDGPU/GlobalISel/legalize-ctpop.mir | 3 +-
.../GlobalISel/legalize-cttz-zero-poison.mir | 3 +-
.../AMDGPU/GlobalISel/legalize-cttz.mir | 3 +-
.../AMDGPU/GlobalISel/legalize-extract.mir | 27 +-
.../AMDGPU/GlobalISel/legalize-fabs.mir | 42 +-
.../AMDGPU/GlobalISel/legalize-fadd.mir | 70 +-
.../GlobalISel/legalize-fcanonicalize.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fceil.mir | 21 +-
.../AMDGPU/GlobalISel/legalize-fcopysign.mir | 20 +-
.../AMDGPU/GlobalISel/legalize-fcos.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fdiv.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fexp.mir | 16 +-
.../AMDGPU/GlobalISel/legalize-fexp2.mir | 14 +-
.../AMDGPU/GlobalISel/legalize-ffloor.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-flog.mir | 7 +-
.../AMDGPU/GlobalISel/legalize-flog10.mir | 7 +-
.../AMDGPU/GlobalISel/legalize-fma.mir | 70 +-
.../AMDGPU/GlobalISel/legalize-fmad.s16.mir | 120 +-
.../AMDGPU/GlobalISel/legalize-fmaxnum.mir | 84 +-
.../AMDGPU/GlobalISel/legalize-fminnum.mir | 84 +-
.../AMDGPU/GlobalISel/legalize-fmul.mir | 70 +-
.../AMDGPU/GlobalISel/legalize-fneg.mir | 30 +-
.../AMDGPU/GlobalISel/legalize-fpow.mir | 14 +-
.../AMDGPU/GlobalISel/legalize-fptrunc.mir | 28 +-
.../AMDGPU/GlobalISel/legalize-freeze.mir | 46 +-
.../AMDGPU/GlobalISel/legalize-fshl.mir | 74 +-
.../AMDGPU/GlobalISel/legalize-fshr.mir | 74 +-
.../AMDGPU/GlobalISel/legalize-fsin.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fsqrt.mir | 40 +-
.../AMDGPU/GlobalISel/legalize-fsub.mir | 70 +-
.../legalize-implicit-def-s1025.mir | 372 +--
.../GlobalISel/legalize-implicit-def.mir | 32 +-
.../AMDGPU/GlobalISel/legalize-insert.mir | 180 +-
.../GlobalISel/legalize-intrinsic-round.mir | 70 +-
.../GlobalISel/legalize-intrinsic-trunc.mir | 21 +-
.../legalize-llvm.amdgcn.image.load.2d.d16.ll | 290 +-
...legalize-llvm.amdgcn.image.store.2d.d16.ll | 15 +-
.../legalize-llvm.amdgcn.s.buffer.load.mir | 27 +-
.../GlobalISel/legalize-load-constant.mir | 280 +-
.../AMDGPU/GlobalISel/legalize-load-flat.mir | 362 +--
.../GlobalISel/legalize-load-global.mir | 1312 ++++----
.../AMDGPU/GlobalISel/legalize-load-local.mir | 560 ++--
.../GlobalISel/legalize-load-private.mir | 532 ++--
.../AMDGPU/GlobalISel/legalize-lshr.mir | 117 +-
.../GlobalISel/legalize-merge-values.mir | 647 ++--
.../AMDGPU/GlobalISel/legalize-mul.mir | 44 +-
.../CodeGen/AMDGPU/GlobalISel/legalize-or.mir | 47 +-
.../AMDGPU/GlobalISel/legalize-phi.mir | 54 +-
.../AMDGPU/GlobalISel/legalize-saddo.mir | 74 +-
.../AMDGPU/GlobalISel/legalize-saddsat.mir | 72 +-
.../AMDGPU/GlobalISel/legalize-sdiv.mir | 18 +-
.../AMDGPU/GlobalISel/legalize-select.mir | 24 +-
.../AMDGPU/GlobalISel/legalize-sext-inreg.mir | 130 +-
.../AMDGPU/GlobalISel/legalize-sext.mir | 25 +-
.../AMDGPU/GlobalISel/legalize-shl.mir | 91 +-
.../GlobalISel/legalize-shuffle-vector.mir | 27 +-
.../legalize-shuffle-vector.s16.mir | 175 +-
.../AMDGPU/GlobalISel/legalize-sitofp.mir | 16 +-
.../AMDGPU/GlobalISel/legalize-smax.mir | 44 +-
.../AMDGPU/GlobalISel/legalize-smin.mir | 44 +-
.../AMDGPU/GlobalISel/legalize-smulh.mir | 63 +-
.../AMDGPU/GlobalISel/legalize-smulo.mir | 59 +-
.../AMDGPU/GlobalISel/legalize-srem.mir | 18 +-
.../AMDGPU/GlobalISel/legalize-sshlsat.mir | 69 +-
.../AMDGPU/GlobalISel/legalize-ssubo.mir | 74 +-
.../AMDGPU/GlobalISel/legalize-ssubsat.mir | 71 +-
.../GlobalISel/legalize-store-global.mir | 36 +-
.../AMDGPU/GlobalISel/legalize-store.mir | 331 +-
.../AMDGPU/GlobalISel/legalize-sub.mir | 44 +-
.../AMDGPU/GlobalISel/legalize-trunc.mir | 66 +-
.../AMDGPU/GlobalISel/legalize-uaddo.mir | 70 +-
.../AMDGPU/GlobalISel/legalize-uaddsat.mir | 47 +-
.../AMDGPU/GlobalISel/legalize-udiv.mir | 16 +-
.../AMDGPU/GlobalISel/legalize-uitofp.mir | 16 +-
.../AMDGPU/GlobalISel/legalize-umax.mir | 28 +-
.../AMDGPU/GlobalISel/legalize-umin.mir | 28 +-
.../AMDGPU/GlobalISel/legalize-umulh.mir | 72 +-
.../AMDGPU/GlobalISel/legalize-umulo.mir | 62 +-
.../AMDGPU/GlobalISel/legalize-urem.mir | 16 +-
.../AMDGPU/GlobalISel/legalize-ushlsat.mir | 47 +-
.../AMDGPU/GlobalISel/legalize-usubo.mir | 70 +-
.../AMDGPU/GlobalISel/legalize-usubsat.mir | 47 +-
.../legalize-vector-args-gfx8-plus.mir | 58 +-
.../AMDGPU/GlobalISel/legalize-xor.mir | 47 +-
.../AMDGPU/GlobalISel/legalize-zext.mir | 53 +-
.../CodeGen/AMDGPU/GlobalISel/llvm.abs.ll | 96 +-
.../llvm.amdgcn.image.load.1d.d16.ll | 130 +-
.../llvm.amdgcn.image.store.2d.d16.ll | 23 +-
.../GlobalISel/llvm.amdgcn.intersect_ray.ll | 266 +-
.../llvm.amdgcn.raw.buffer.load.format.f16.ll | 267 +-
...m.amdgcn.raw.ptr.buffer.load.format.f16.ll | 166 +-
.../llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll | 105 +-
.../llvm.amdgcn.raw.tbuffer.load.f16.ll | 206 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll | 54 +-
.../GlobalISel/llvm.amdgcn.set.inactive.ll | 104 +-
...vm.amdgcn.struct.buffer.load.format.f16.ll | 205 +-
...mdgcn.struct.ptr.buffer.load.format.f16.ll | 205 +-
...llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll | 205 +-
.../llvm.amdgcn.struct.tbuffer.load.f16.ll | 205 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll | 89 +-
.../GlobalISel/llvm.amdgcn.wqm.demote.ll | 74 +-
.../CodeGen/AMDGPU/GlobalISel/llvm.memset.ll | 137 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll | 373 +--
llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll | 34 +-
.../CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll | 20 +-
...ctlz-from-umul-to-lshr-in-postlegalizer.ll | 97 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll | 251 +-
.../test/CodeGen/AMDGPU/GlobalISel/saddsat.ll | 2579 +++++++++------
.../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 2 +-
.../CodeGen/AMDGPU/GlobalISel/sext_inreg.ll | 315 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll | 343 +-
llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll | 14 +-
.../test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll | 2604 +++++++++------
.../AMDGPU/GlobalISel/strict_fma.f16.ll | 693 ++--
llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll | 29 +-
.../CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll | 114 +-
.../test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll | 1012 +++---
.../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 1 -
llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll | 14 +-
.../test/CodeGen/AMDGPU/GlobalISel/usubsat.ll | 973 ++++--
.../AMDGPU/GlobalISel/vni8-across-blocks.ll | 62 +-
...ffer-fat-pointers-contents-legalization.ll | 293 +-
.../build-vector-packed-partial-undef.ll | 234 +-
llvm/test/CodeGen/AMDGPU/ctpop16.ll | 382 ++-
.../CodeGen/AMDGPU/dagcombine-fmul-sel.ll | 124 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 96 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 186 +-
llvm/test/CodeGen/AMDGPU/fmed3.ll | 214 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 186 +-
llvm/test/CodeGen/AMDGPU/fptoi.i128.ll | 602 +++-
llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll | 685 ++--
llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll | 566 +++-
llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll | 57 +-
llvm/test/CodeGen/AMDGPU/fptrunc.ll | 6 +-
llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll | 56 +-
.../CodeGen/AMDGPU/integer-mad-patterns.ll | 1202 ++++---
.../AMDGPU/llvm.amdgcn.intersect_ray.ll | 314 +-
.../AMDGPU/llvm.amdgcn.readfirstlane.ll | 142 +-
llvm/test/CodeGen/AMDGPU/llvm.exp.ll | 690 ++--
llvm/test/CodeGen/AMDGPU/llvm.exp10.ll | 852 +++--
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll | 500 ++-
llvm/test/CodeGen/AMDGPU/llvm.frexp.ll | 203 +-
llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll | 597 ++--
llvm/test/CodeGen/AMDGPU/llvm.log.ll | 1261 +++++---
llvm/test/CodeGen/AMDGPU/llvm.log10.ll | 1261 +++++---
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 712 +++--
llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll | 79 +-
llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll | 903 ++++--
.../AMDGPU/memset-param-combinations.ll | 2814 +++++++++++------
llvm/test/CodeGen/AMDGPU/roundeven.ll | 132 +-
.../CodeGen/AMDGPU/shrink-add-sub-constant.ll | 2103 ++++++++----
llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll | 178 +-
llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll | 178 +-
llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll | 187 +-
llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll | 138 +-
llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll | 75 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll | 56 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll | 54 +-
.../test/CodeGen/AMDGPU/vector-reduce-umin.ll | 8 +-
llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll | 66 +-
192 files changed, 28998 insertions(+), 19786 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 1577bb5b53d20..a911697ba73f2 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -9342,7 +9342,7 @@ LegalizerHelper::lowerMergeValues(MachineInstr &MI) {
auto [DstReg, DstTy, Src0Reg, Src0Ty] = MI.getFirst2RegLLTs();
unsigned PartSize = Src0Ty.getSizeInBits();
- LLT WideTy = LLT::scalar(DstTy.getSizeInBits());
+ LLT WideTy = LLT::integer(DstTy.getSizeInBits());
Register ResultReg = MIRBuilder.buildZExt(WideTy, Src0Reg).getReg(0);
for (unsigned I = 2; I != NumOps; ++I) {
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll
index 3f95b41548c9a..763370705ebda 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
@@ -200,12 +200,13 @@ define <2 x i16> @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX7-LABEL: s_add_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s4, s17, 16
-; GFX7-NEXT: s_lshr_b32 s5, s16, 16
-; GFX7-NEXT: s_add_i32 s5, s5, s4
+; GFX7-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-NEXT: s_add_i32 s4, s4, s5
; GFX7-NEXT: s_add_i32 s16, s16, s17
-; GFX7-NEXT: s_lshl_b32 s4, s5, 16
+; GFX7-NEXT: s_and_b32 s4, s4, 0xffff
; GFX7-NEXT: s_and_b32 s5, s16, 0xffff
+; GFX7-NEXT: s_lshl_b32 s4, s4, 16
; GFX7-NEXT: s_or_b32 s4, s5, s4
; GFX7-NEXT: v_mov_b32_e32 v0, s4
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -224,12 +225,13 @@ define <2 x i16> @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX8-LABEL: s_add_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s17, 16
-; GFX8-NEXT: s_lshr_b32 s5, s16, 16
-; GFX8-NEXT: s_add_i32 s5, s5, s4
+; GFX8-NEXT: s_lshr_b32 s4, s16, 16
+; GFX8-NEXT: s_lshr_b32 s5, s17, 16
+; GFX8-NEXT: s_add_i32 s4, s4, s5
; GFX8-NEXT: s_add_i32 s16, s16, s17
-; GFX8-NEXT: s_lshl_b32 s4, s5, 16
-; GFX8-NEXT: s_and_b32 s5, s16, 0xffff
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_and_b32 s5, 0xffff, s16
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
; GFX8-NEXT: s_or_b32 s4, s5, s4
; GFX8-NEXT: v_mov_b32_e32 v0, s4
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -281,13 +283,14 @@ define <2 x i16> @v_add_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX7-LABEL: v_add_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16:
@@ -299,9 +302,9 @@ define <2 x i16> @v_add_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX8-LABEL: v_add_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
index aa7024fdf1245..a050fc43b453c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.v2i16.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
@@ -9,13 +9,14 @@ define <2 x i16> @v_add_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX7-LABEL: v_add_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16:
@@ -27,9 +28,9 @@ define <2 x i16> @v_add_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX8-LABEL: v_add_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16:
@@ -46,13 +47,14 @@ define <2 x i16> @v_add_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, v3, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16_fneg_lhs:
@@ -65,9 +67,9 @@ define <2 x i16> @v_add_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX8-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16_fneg_lhs:
@@ -88,11 +90,12 @@ define <2 x i16> @v_add_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX7-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX7-NEXT: v_add_i32_e32 v2, vcc, v2, v3
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16_fneg_rhs:
@@ -105,9 +108,9 @@ define <2 x i16> @v_add_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
-; GFX8-NEXT: v_add_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16_fneg_rhs:
@@ -131,6 +134,7 @@ define <2 x i16> @v_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-NEXT: v_add_i32_e32 v1, vcc, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
@@ -169,10 +173,13 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_splat(<2 x i16> %a) {
; GFX7-LABEL: v_add_v2i16_neg_inline_imm_splat:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v1, vcc, 64, v0
-; GFX7-NEXT: s_mov_b32 s4, 0xffff0000
-; GFX7-NEXT: v_bfi_b32 v0, s4, v0, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xffc00000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xffffffc0, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xffffffc0, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16_neg_inline_imm_splat:
@@ -184,10 +191,10 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_splat(<2 x i16> %a) {
; GFX8-LABEL: v_add_v2i16_neg_inline_imm_splat:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 64
-; GFX8-NEXT: v_sub_u16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_subrev_u16_e32 v0, 64, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_not_b32_e32 v1, 63
+; GFX8-NEXT: v_add_u16_e32 v2, 0xffc0, v0
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16_neg_inline_imm_splat:
@@ -203,10 +210,13 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_lo(<2 x i16> %a) {
; GFX7-LABEL: v_add_v2i16_neg_inline_imm_lo:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_subrev_i32_e32 v1, vcc, 64, v0
-; GFX7-NEXT: s_mov_b32 s4, 0xffff0000
-; GFX7-NEXT: v_bfi_b32 v0, s4, v0, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0x40000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, 4, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xffffffc0, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16_neg_inline_imm_lo:
@@ -219,10 +229,10 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_lo(<2 x i16> %a) {
; GFX8-LABEL: v_add_v2i16_neg_inline_imm_lo:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 4
-; GFX8-NEXT: v_add_u16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_subrev_u16_e32 v0, 64, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 4
+; GFX8-NEXT: v_add_u16_e32 v1, 0xffc0, v0
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16_neg_inline_imm_lo:
@@ -238,10 +248,13 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
; GFX7-LABEL: v_add_v2i16_neg_inline_imm_hi:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, 4, v0
-; GFX7-NEXT: s_mov_b32 s4, 0xffff0000
-; GFX7-NEXT: v_bfi_b32 v0, s4, v0, v1
-; GFX7-NEXT: v_add_i32_e32 v0, vcc, 0xffc00000, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_add_i32_e32 v1, vcc, 0xffffffc0, v1
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 4, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_add_v2i16_neg_inline_imm_hi:
@@ -254,10 +267,10 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
; GFX8-LABEL: v_add_v2i16_neg_inline_imm_hi:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, 64
-; GFX8-NEXT: v_add_u16_e32 v1, 4, v0
-; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_not_b32_e32 v1, 63
+; GFX8-NEXT: v_add_u16_e32 v2, 4, v0
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_add_v2i16_neg_inline_imm_hi:
@@ -272,11 +285,13 @@ define <2 x i16> @v_add_v2i16_neg_inline_imm_hi(<2 x i16> %a) {
define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
; GFX7-LABEL: s_add_v2i16_neg_inline_imm_splat:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX7-NEXT: s_lshr_b32 s1, s0, 16
+; GFX7-NEXT: s_sub_i32 s1, s1, 64
; GFX7-NEXT: s_sub_i32 s0, s0, 64
+; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s1, s0
-; GFX7-NEXT: s_add_i32 s0, s0, 0xffc00000
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_add_v2i16_neg_inline_imm_splat:
@@ -289,11 +304,13 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
;
; GFX8-LABEL: s_add_v2i16_neg_inline_imm_splat:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_add_i32 s1, s1, 0xffc0
; GFX8-NEXT: s_add_i32 s0, s0, 0xffc0
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_add_i32 s0, s0, 0xffc00000
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16_neg_inline_imm_splat:
@@ -311,11 +328,13 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
; GFX7-LABEL: s_add_v2i16_neg_inline_imm_lo:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX7-NEXT: s_lshr_b32 s1, s0, 16
+; GFX7-NEXT: s_add_i32 s1, s1, 4
; GFX7-NEXT: s_sub_i32 s0, s0, 64
+; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s1, s0
-; GFX7-NEXT: s_add_i32 s0, s0, 0x40000
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_add_v2i16_neg_inline_imm_lo:
@@ -328,11 +347,13 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
;
; GFX8-LABEL: s_add_v2i16_neg_inline_imm_lo:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_add_i32 s1, s1, 4
; GFX8-NEXT: s_add_i32 s0, s0, 0xffc0
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_add_i32 s0, s0, 0x40000
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16_neg_inline_imm_lo:
@@ -350,11 +371,13 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
; GFX7-LABEL: s_add_v2i16_neg_inline_imm_hi:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX7-NEXT: s_lshr_b32 s1, s0, 16
+; GFX7-NEXT: s_sub_i32 s1, s1, 64
; GFX7-NEXT: s_add_i32 s0, s0, 4
+; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s1, s0
-; GFX7-NEXT: s_add_i32 s0, s0, 0xffc00000
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_add_v2i16_neg_inline_imm_hi:
@@ -367,11 +390,13 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
;
; GFX8-LABEL: s_add_v2i16_neg_inline_imm_hi:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_add_i32 s1, s1, 0xffc0
; GFX8-NEXT: s_add_i32 s0, s0, 4
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_add_i32 s0, s0, 0xffc00000
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16_neg_inline_imm_hi:
@@ -389,13 +414,14 @@ define amdgpu_ps i32 @s_add_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
define amdgpu_ps i32 @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
; GFX7-LABEL: s_add_v2i16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s2, s1, 16
-; GFX7-NEXT: s_lshr_b32 s3, s0, 16
-; GFX7-NEXT: s_add_i32 s3, s3, s2
+; GFX7-NEXT: s_lshr_b32 s2, s0, 16
+; GFX7-NEXT: s_lshr_b32 s3, s1, 16
+; GFX7-NEXT: s_add_i32 s2, s2, s3
; GFX7-NEXT: s_add_i32 s0, s0, s1
-; GFX7-NEXT: s_lshl_b32 s2, s3, 16
+; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s0, s2
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_add_v2i16:
@@ -409,13 +435,14 @@ define amdgpu_ps i32 @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
;
; GFX8-LABEL: s_add_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_add_i32 s3, s3, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_add_i32 s2, s2, s3
; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s3, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16:
@@ -434,14 +461,17 @@ define amdgpu_ps i32 @s_add_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
define amdgpu_ps i32 @s_add_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg %b) {
; GFX7-LABEL: s_add_v2i16_fneg_lhs:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX7-NEXT: s_lshr_b32 s2, s1, 16
-; GFX7-NEXT: s_lshr_b32 s3, s0, 16
-; GFX7-NEXT: s_add_i32 s3, s3, s2
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: s_lshr_b32 s2, s0, 16
+; GFX7-NEXT: s_lshr_b32 s3, s1, 16
+; GFX7-NEXT: s_add_i32 s2, s2, s3
; GFX7-NEXT: s_add_i32 s0, s0, s1
-; GFX7-NEXT: s_lshl_b32 s2, s3, 16
+; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s0, s2
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_add_v2i16_fneg_lhs:
@@ -458,14 +488,17 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
;
; GFX8-LABEL: s_add_v2i16_fneg_lhs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_add_i32 s3, s3, s2
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_add_i32 s2, s2, s3
; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s3, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16_fneg_lhs:
@@ -488,14 +521,17 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
define amdgpu_ps i32 @s_add_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg %b) {
; GFX7-LABEL: s_add_v2i16_fneg_rhs:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_xor_b32 s1, s1, 0x80008000
+; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX7-NEXT: v_readfirstlane_b32 s1, v0
; GFX7-NEXT: s_lshr_b32 s2, s0, 16
; GFX7-NEXT: s_lshr_b32 s3, s1, 16
; GFX7-NEXT: s_add_i32 s2, s2, s3
; GFX7-NEXT: s_add_i32 s0, s0, s1
-; GFX7-NEXT: s_lshl_b32 s2, s2, 16
+; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_or_b32 s0, s0, s2
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_add_v2i16_fneg_rhs:
@@ -512,14 +548,17 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
;
; GFX8-LABEL: s_add_v2i16_fneg_rhs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x80008000
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
; GFX8-NEXT: s_lshr_b32 s2, s0, 16
; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_add_i32 s2, s2, s3
; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16_fneg_rhs:
@@ -542,14 +581,19 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
define amdgpu_ps i32 @s_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x half> inreg %b) {
; GFX7-LABEL: s_add_v2i16_fneg_lhs_fneg_rhs:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX7-NEXT: s_xor_b32 s1, s1, 0x80008000
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s1
+; GFX7-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX7-NEXT: v_readfirstlane_b32 s1, v0
; GFX7-NEXT: s_lshr_b32 s2, s0, 16
; GFX7-NEXT: s_lshr_b32 s3, s1, 16
-; GFX7-NEXT: s_add_i32 s0, s0, s1
; GFX7-NEXT: s_add_i32 s2, s2, s3
+; GFX7-NEXT: s_add_i32 s0, s0, s1
+; GFX7-NEXT: s_and_b32 s1, s2, 0xffff
; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX7-NEXT: s_lshl_b32 s1, s2, 16
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
@@ -570,15 +614,20 @@ define amdgpu_ps i32 @s_add_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x ha
;
; GFX8-LABEL: s_add_v2i16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX8-NEXT: s_add_i32 s2, s0, s1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_add_i32 s2, s2, s3
; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s2, s0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_add_v2i16_fneg_lhs_fneg_rhs:
@@ -606,9 +655,11 @@ define <2 x i16> @add_inline_imm_neg1_0(<2 x i16> %x) {
; GFX7-LABEL: add_inline_imm_neg1_0:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, -1, v0
-; GFX7-NEXT: s_mov_b32 s4, 0xffff
-; GFX7-NEXT: v_bfi_b32 v0, s4, v1, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, -1, v0
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_inline_imm_neg1_0:
@@ -620,8 +671,9 @@ define <2 x i16> @add_inline_imm_neg1_0(<2 x i16> %x) {
; GFX8-LABEL: add_inline_imm_neg1_0:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX8-NEXT: v_add_u16_e32 v0, -1, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -638,9 +690,11 @@ define <2 x i16> @add_inline_imm_1_0(<2 x i16> %x) {
; GFX7-LABEL: add_inline_imm_1_0:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX7-NEXT: s_mov_b32 s4, 0xffff
-; GFX7-NEXT: v_bfi_b32 v0, s4, v1, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_inline_imm_1_0:
@@ -652,8 +706,9 @@ define <2 x i16> @add_inline_imm_1_0(<2 x i16> %x) {
; GFX8-LABEL: add_inline_imm_1_0:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX8-NEXT: v_add_u16_e32 v0, 1, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
index bfe527096d634..3a8d2a90a7a84 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/add.vni16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
; FIXME: codegen regression, related to:
@@ -81,10 +81,10 @@ define <3 x i16> @add_v3i16_arg(<3 x i16> %a, <3 x i16> %b) {
; GFX8-LABEL: add_v3i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
+; GFX8-NEXT: v_add_u16_e32 v4, v0, v2
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
; GFX8-NEXT: v_add_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v3i16_arg:
@@ -104,12 +104,12 @@ define void @add_v4i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v6, v1, v3
-; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v3, v0, v2
+; GFX8-NEXT: v_add_u16_e32 v6, v0, v2
; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v1, v6, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX8-NEXT: v_add_u16_e32 v2, v1, v3
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -136,12 +136,12 @@ define <4 x i16> @add_v4i16_arg(<4 x i16> %a, <4 x i16> %b) {
; GFX8-LABEL: add_v4i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT: v_add_u16_e32 v4, v0, v2
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v2, v1, v3
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v4i16_arg:
@@ -266,13 +266,13 @@ define <5 x i16> @add_v5i16_arg(<5 x i16> %a, <5 x i16> %b) {
; GFX8-LABEL: add_v5i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v6, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v7, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v3
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v4
+; GFX8-NEXT: v_add_u16_e32 v6, v0, v3
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v3, v1, v4
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
; GFX8-NEXT: v_add_u16_e32 v2, v2, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v7
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v6
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v5i16_arg:
@@ -293,15 +293,15 @@ define void @add_v6i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX8-NEXT: flat_load_dwordx3 v[6:8], v[0:1]
; GFX8-NEXT: flat_load_dwordx3 v[0:2], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v3, v8, v2
-; GFX8-NEXT: v_add_u16_sdwa v2, v8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v8, v7, v1
-; GFX8-NEXT: v_add_u16_sdwa v1, v7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v7, v6, v0
+; GFX8-NEXT: v_add_u16_e32 v3, v6, v0
; GFX8-NEXT: v_add_u16_sdwa v0, v6, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v8, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v7, v0
+; GFX8-NEXT: v_add_u16_e32 v6, v7, v1
+; GFX8-NEXT: v_add_u16_sdwa v1, v7, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v7, v8, v2
+; GFX8-NEXT: v_add_u16_sdwa v2, v8, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v7, v2
; GFX8-NEXT: flat_store_dwordx3 v[4:5], v[0:2]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -329,15 +329,15 @@ define <6 x i16> @add_v6i16_arg(<6 x i16> %a, <6 x i16> %b) {
; GFX8-LABEL: add_v6i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v6, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v7, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v8, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v3
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v4
-; GFX8-NEXT: v_add_u16_e32 v2, v2, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT: v_add_u16_e32 v6, v0, v3
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v3, v1, v4
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_add_u16_e32 v3, v2, v5
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v6i16_arg:
@@ -500,16 +500,16 @@ define <7 x i16> @add_v7i16_arg(<7 x i16> %a, <7 x i16> %b) {
; GFX8-LABEL: add_v7i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v8, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v9, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v10, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_add_u16_e32 v2, v2, v6
+; GFX8-NEXT: v_add_u16_e32 v8, v0, v4
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v4, v1, v5
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_add_u16_e32 v4, v2, v6
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
; GFX8-NEXT: v_add_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v10
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v8
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v7i16_arg:
@@ -537,24 +537,24 @@ define void @add_v9i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addrs
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_ushort v0, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_add_u16_sdwa v15, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v10, v6, v10
+; GFX8-NEXT: v_add_u16_e32 v1, v6, v10
+; GFX8-NEXT: v_add_u16_sdwa v2, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v3, v7, v11
+; GFX8-NEXT: v_add_u16_sdwa v10, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v11, v8, v12
+; GFX8-NEXT: v_add_u16_sdwa v8, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v12, v9, v13
+; GFX8-NEXT: v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_u32_e32 v6, vcc, 16, v4
-; GFX8-NEXT: v_add_u16_sdwa v1, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v2, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v3, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v9, v9, v13
-; GFX8-NEXT: v_add_u16_e32 v8, v8, v12
-; GFX8-NEXT: v_add_u16_e32 v11, v7, v11
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v12, v14, v0
-; GFX8-NEXT: v_or_b32_e32 v3, v9, v3
-; GFX8-NEXT: v_or_b32_e32 v2, v8, v2
-; GFX8-NEXT: v_or_b32_e32 v1, v11, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v10, v15
-; GFX8-NEXT: flat_store_short v[6:7], v12
+; GFX8-NEXT: v_add_u16_e32 v13, v14, v0
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v10
+; GFX8-NEXT: v_or_b32_e32 v2, v11, v8
+; GFX8-NEXT: v_or_b32_e32 v3, v12, v9
+; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: flat_store_short v[6:7], v13
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -592,19 +592,19 @@ define <9 x i16> @add_v9i16_arg(<9 x i16> %a, <9 x i16> %b) {
; GFX8-LABEL: add_v9i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v10, v3, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v11, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v12, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v13, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v5
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v6
-; GFX8-NEXT: v_add_u16_e32 v2, v2, v7
-; GFX8-NEXT: v_add_u16_e32 v3, v3, v8
+; GFX8-NEXT: v_add_u16_e32 v10, v0, v5
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v5, v1, v6
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v5, v1
+; GFX8-NEXT: v_add_u16_e32 v5, v2, v7
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: v_add_u16_e32 v5, v3, v8
+; GFX8-NEXT: v_add_u16_sdwa v3, v3, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v10, v0
+; GFX8-NEXT: v_or_b32_e32 v3, v5, v3
; GFX8-NEXT: v_add_u16_e32 v4, v4, v9
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v13
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v12
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v11
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v10
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v9i16_arg:
@@ -628,31 +628,31 @@ define void @add_v10i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX8-NEXT: flat_load_dwordx4 v[10:13], v[2:3]
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v0
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, 16, v2
-; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_dword v14, v[0:1]
-; GFX8-NEXT: flat_load_dword v15, v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v2
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_dword v15, v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_add_u16_e32 v0, v9, v13
-; GFX8-NEXT: v_add_u16_sdwa v1, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v2, v8, v12
-; GFX8-NEXT: v_add_u16_sdwa v8, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v9, v7, v11
-; GFX8-NEXT: v_add_u16_sdwa v11, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v12, v6, v10
-; GFX8-NEXT: v_add_u16_sdwa v10, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 16, v4
-; GFX8-NEXT: v_or_b32_e32 v3, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v9, v11
+; GFX8-NEXT: v_add_u16_e32 v0, v6, v10
+; GFX8-NEXT: v_add_u16_sdwa v1, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v2, v7, v11
+; GFX8-NEXT: v_add_u16_sdwa v3, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v6, v8, v12
+; GFX8-NEXT: v_add_u16_sdwa v7, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v8, v9, v13
+; GFX8-NEXT: v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v7
+; GFX8-NEXT: v_or_b32_e32 v3, v8, v9
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v8, v14, v15
-; GFX8-NEXT: v_add_u16_sdwa v9, v14, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
-; GFX8-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX8-NEXT: v_or_b32_e32 v0, v12, v10
-; GFX8-NEXT: flat_store_dword v[6:7], v8
+; GFX8-NEXT: v_add_u16_e32 v6, v14, v15
+; GFX8-NEXT: v_add_u16_sdwa v7, v14, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v4
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
+; GFX8-NEXT: flat_store_dword v[0:1], v6
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -687,44 +687,53 @@ define void @add_v11i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx4 v[6:9], v[0:1]
; GFX8-NEXT: flat_load_dwordx4 v[10:13], v[2:3]
-; GFX8-NEXT: s_waitcnt vmcnt(0)
+; GFX8-NEXT: v_add_u32_e32 v14, vcc, 16, v2
+; GFX8-NEXT: v_addc_u32_e32 v15, vcc, 0, v3, vcc
+; GFX8-NEXT: v_add_u32_e32 v16, vcc, 18, v2
+; GFX8-NEXT: v_addc_u32_e32 v17, vcc, 0, v3, vcc
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 20, v2
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; GFX8-NEXT: flat_load_ushort v14, v[14:15]
+; GFX8-NEXT: flat_load_ushort v15, v[16:17]
+; GFX8-NEXT: flat_load_ushort v16, v[2:3]
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; GFX8-NEXT: s_waitcnt vmcnt(3)
; GFX8-NEXT: v_add_u16_e32 v17, v6, v10
; GFX8-NEXT: v_add_u16_sdwa v10, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 20, v0
-; GFX8-NEXT: v_add_u16_e32 v16, v7, v11
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 18, v0
+; GFX8-NEXT: v_add_u16_e32 v18, v7, v11
; GFX8-NEXT: v_add_u16_sdwa v11, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v0
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 20, v0
+; GFX8-NEXT: flat_load_ushort v2, v[2:3]
+; GFX8-NEXT: flat_load_ushort v3, v[6:7]
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-NEXT: v_add_u16_e32 v15, v8, v12
-; GFX8-NEXT: v_add_u16_sdwa v12, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v14, v9, v13
-; GFX8-NEXT: v_add_u16_sdwa v13, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_load_ushort v18, v[6:7]
-; GFX8-NEXT: flat_load_dword v19, v[0:1]
-; GFX8-NEXT: flat_load_dword v20, v[8:9]
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 20, v2
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
; GFX8-NEXT: flat_load_ushort v21, v[0:1]
-; GFX8-NEXT: v_add_u32_e32 v6, vcc, 20, v4
+; GFX8-NEXT: v_add_u32_e32 v6, vcc, 16, v4
; GFX8-NEXT: v_addc_u32_e32 v7, vcc, 0, v5, vcc
-; GFX8-NEXT: v_add_u32_e32 v8, vcc, 16, v4
-; GFX8-NEXT: v_or_b32_e32 v2, v15, v12
-; GFX8-NEXT: v_or_b32_e32 v1, v16, v11
-; GFX8-NEXT: v_or_b32_e32 v0, v17, v10
+; GFX8-NEXT: v_add_u16_e32 v19, v8, v12
+; GFX8-NEXT: v_add_u16_sdwa v12, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u32_e32 v8, vcc, 18, v4
+; GFX8-NEXT: v_add_u16_e32 v20, v9, v13
+; GFX8-NEXT: v_add_u16_sdwa v13, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_addc_u32_e32 v9, vcc, 0, v5, vcc
-; GFX8-NEXT: v_or_b32_e32 v3, v14, v13
+; GFX8-NEXT: v_or_b32_e32 v0, v17, v10
+; GFX8-NEXT: v_or_b32_e32 v1, v18, v11
+; GFX8-NEXT: v_add_u32_e32 v10, vcc, 20, v4
+; GFX8-NEXT: v_addc_u32_e32 v11, vcc, 0, v5, vcc
+; GFX8-NEXT: s_waitcnt vmcnt(2)
+; GFX8-NEXT: v_add_u16_e32 v14, v2, v14
; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_add_u16_e32 v10, v19, v20
-; GFX8-NEXT: v_add_u16_sdwa v11, v19, v20 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v10, v10, v11
+; GFX8-NEXT: v_add_u16_e32 v15, v3, v15
+; GFX8-NEXT: v_or_b32_e32 v2, v19, v12
+; GFX8-NEXT: v_or_b32_e32 v3, v20, v13
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v12, v18, v21
-; GFX8-NEXT: flat_store_short v[6:7], v12
-; GFX8-NEXT: flat_store_dword v[8:9], v10
+; GFX8-NEXT: v_add_u16_e32 v16, v21, v16
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT: flat_store_short v[6:7], v14
+; GFX8-NEXT: flat_store_short v[8:9], v15
+; GFX8-NEXT: flat_store_short v[10:11], v16
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
@@ -778,22 +787,22 @@ define <11 x i16> @add_v11i16_arg(<11 x i16> %a, <11 x i16> %b) {
; GFX8-LABEL: add_v11i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v12, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v13, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v14, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v15, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v16, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v6
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v7
-; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
-; GFX8-NEXT: v_add_u16_e32 v3, v3, v9
-; GFX8-NEXT: v_add_u16_e32 v4, v4, v10
+; GFX8-NEXT: v_add_u16_e32 v12, v0, v6
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v6, v1, v7
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX8-NEXT: v_add_u16_e32 v6, v2, v8
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: v_add_u16_e32 v6, v3, v9
+; GFX8-NEXT: v_add_u16_sdwa v3, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
+; GFX8-NEXT: v_add_u16_e32 v6, v4, v10
+; GFX8-NEXT: v_add_u16_sdwa v4, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v12, v0
+; GFX8-NEXT: v_or_b32_e32 v4, v6, v4
; GFX8-NEXT: v_add_u16_e32 v5, v5, v11
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v16
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v15
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v14
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v13
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v12
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v11i16_arg:
@@ -816,34 +825,34 @@ define void @add_v12i16(ptr addrspace(1) %ptra, ptr addrspace(1) %ptrb, ptr addr
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: flat_load_dwordx4 v[6:9], v[0:1]
; GFX8-NEXT: flat_load_dwordx4 v[10:13], v[2:3]
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v0
-; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 16, v2
; GFX8-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; GFX8-NEXT: flat_load_dwordx2 v[2:3], v[2:3]
-; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_add_u16_e32 v14, v9, v13
-; GFX8-NEXT: v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v13, v8, v12
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v0
+; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT: flat_load_dwordx2 v[14:15], v[2:3]
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_add_u16_e32 v2, v6, v10
+; GFX8-NEXT: v_add_u16_sdwa v3, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v10, v7, v11
+; GFX8-NEXT: v_add_u16_sdwa v11, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: flat_load_dwordx2 v[6:7], v[0:1]
+; GFX8-NEXT: v_add_u16_e32 v16, v8, v12
; GFX8-NEXT: v_add_u16_sdwa v8, v8, v12 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v12, v7, v11
-; GFX8-NEXT: v_add_u16_sdwa v7, v7, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v11, v6, v10
-; GFX8-NEXT: v_add_u16_sdwa v6, v6, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v10, v1, v3
-; GFX8-NEXT: v_add_u16_sdwa v15, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v16, v0, v2
-; GFX8-NEXT: v_add_u16_sdwa v17, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v3, v14, v9
-; GFX8-NEXT: v_or_b32_e32 v2, v13, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v12, v7
-; GFX8-NEXT: v_or_b32_e32 v0, v11, v6
+; GFX8-NEXT: v_add_u16_e32 v12, v9, v13
+; GFX8-NEXT: v_add_u16_sdwa v9, v9, v13 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v3
+; GFX8-NEXT: v_or_b32_e32 v1, v10, v11
+; GFX8-NEXT: v_or_b32_e32 v2, v16, v8
+; GFX8-NEXT: v_or_b32_e32 v3, v12, v9
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT: v_or_b32_e32 v7, v10, v15
+; GFX8-NEXT: s_waitcnt vmcnt(1)
+; GFX8-NEXT: v_add_u16_e32 v8, v6, v14
+; GFX8-NEXT: v_add_u16_sdwa v6, v6, v14 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v9, v7, v15
+; GFX8-NEXT: v_add_u16_sdwa v7, v7, v15 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, 16, v4
-; GFX8-NEXT: v_or_b32_e32 v6, v16, v17
+; GFX8-NEXT: v_or_b32_e32 v6, v8, v6
+; GFX8-NEXT: v_or_b32_e32 v7, v9, v7
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v5, vcc
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[6:7]
; GFX8-NEXT: s_waitcnt vmcnt(0)
@@ -879,24 +888,24 @@ define <12 x i16> @add_v12i16_arg(<12 x i16> %a, <12 x i16> %b) {
; GFX8-LABEL: add_v12i16_arg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v12, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v13, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v14, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v15, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v16, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_sdwa v17, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e32 v0, v0, v6
-; GFX8-NEXT: v_add_u16_e32 v1, v1, v7
-; GFX8-NEXT: v_add_u16_e32 v2, v2, v8
-; GFX8-NEXT: v_add_u16_e32 v3, v3, v9
-; GFX8-NEXT: v_add_u16_e32 v4, v4, v10
-; GFX8-NEXT: v_add_u16_e32 v5, v5, v11
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v17
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v16
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v15
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v14
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v13
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v12
+; GFX8-NEXT: v_add_u16_e32 v12, v0, v6
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v6, v1, v7
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX8-NEXT: v_add_u16_e32 v6, v2, v8
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v2, v6, v2
+; GFX8-NEXT: v_add_u16_e32 v6, v3, v9
+; GFX8-NEXT: v_add_u16_sdwa v3, v3, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
+; GFX8-NEXT: v_add_u16_e32 v6, v4, v10
+; GFX8-NEXT: v_add_u16_sdwa v4, v4, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v4, v6, v4
+; GFX8-NEXT: v_add_u16_e32 v6, v5, v11
+; GFX8-NEXT: v_add_u16_sdwa v5, v5, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v12, v0
+; GFX8-NEXT: v_or_b32_e32 v5, v6, v5
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: add_v12i16_arg:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
index 83fbf82b46811..c94425b8897f8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/andn2.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; FIXME: regbankcombiner regression, related to:
; - looking through copy and splitting G_CONSTANT i64 to two i32 constants
@@ -700,31 +700,49 @@ define <2 x i16> @v_andn2_v2i16(<2 x i16> %src0, <2 x i16> %src1) {
define amdgpu_ps i48 @s_andn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX6-LABEL: s_andn2_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_andn2_b32 s0, s3, s5
-; GFX6-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX6-NEXT: s_andn2_b32 s0, s2, s4
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_mov_b32 s0, -1
+; GFX6-NEXT: s_mov_b32 s1, 0xffff
+; GFX6-NEXT: s_or_b32 s4, s4, s7
+; GFX6-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX6-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_andn2_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s4, 16
-; GFX9-NEXT: s_lshr_b32 s5, s2, 16
-; GFX9-NEXT: s_andn2_b32 s0, s5, s0
-; GFX9-NEXT: s_andn2_b32 s2, s2, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
-; GFX9-NEXT: s_and_b32 s1, s3, s1
+; GFX9-NEXT: s_mov_b64 s[0:1], -1
+; GFX9-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_andn2_v3i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s0, s4, 16
-; GFX10-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10-NEXT: s_andn2_b32 s0, s1, s0
-; GFX10-NEXT: s_andn2_b32 s1, s2, s4
-; GFX10-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX10-NEXT: s_and_b32 s1, s3, s2
+; GFX10-NEXT: s_mov_b64 s[0:1], -1
+; GFX10-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_andn2_v3i16:
@@ -738,13 +756,14 @@ define amdgpu_ps i48 @s_andn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1
;
; GFX11-FAKE16-LABEL: s_andn2_v3i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, s2, s4
-; GFX11-FAKE16-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s3, s2
+; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-FAKE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-FAKE16-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %src0, %not.src1
@@ -755,25 +774,49 @@ define amdgpu_ps i48 @s_andn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1
define amdgpu_ps i48 @s_andn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX6-LABEL: s_andn2_v3i16_commute:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_andn2_b32 s0, s3, s5
-; GFX6-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX6-NEXT: s_andn2_b32 s0, s2, s4
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_mov_b32 s0, -1
+; GFX6-NEXT: s_mov_b32 s1, 0xffff
+; GFX6-NEXT: s_or_b32 s4, s4, s7
+; GFX6-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX6-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_andn2_v3i16_commute:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX9-NEXT: s_andn2_b32 s0, s2, s4
-; GFX9-NEXT: s_pack_lh_b32_b16 s0, s0, s0
-; GFX9-NEXT: s_and_b32 s1, s1, s3
+; GFX9-NEXT: s_mov_b64 s[0:1], -1
+; GFX9-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_andn2_v3i16_commute:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_andn2_b32 s0, s2, s4
-; GFX10-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX10-NEXT: s_pack_lh_b32_b16 s0, s0, s0
-; GFX10-NEXT: s_and_b32 s1, s1, s3
+; GFX10-NEXT: s_mov_b64 s[0:1], -1
+; GFX10-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_andn2_v3i16_commute:
@@ -787,10 +830,14 @@ define amdgpu_ps i48 @s_andn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inr
;
; GFX11-FAKE16-LABEL: s_andn2_v3i16_commute:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s0, s2, s4
-; GFX11-FAKE16-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX11-FAKE16-NEXT: s_pack_lh_b32_b16 s0, s0, s0
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, s3
+; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-FAKE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-FAKE16-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %not.src1, %src0
@@ -801,52 +848,64 @@ define amdgpu_ps i48 @s_andn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inr
define amdgpu_ps { i48, i48 } @s_andn2_v3i16_multi_use(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX6-LABEL: s_andn2_v3i16_multi_use:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s1, s4, 16
-; GFX6-NEXT: s_lshr_b32 s0, s2, 16
-; GFX6-NEXT: s_andn2_b32 s6, 0xffff, s4
-; GFX6-NEXT: s_not_b32 s4, s1
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
-; GFX6-NEXT: s_andn2_b32 s0, s0, s1
-; GFX6-NEXT: s_or_b32 s4, s6, s4
-; GFX6-NEXT: s_andn2_b32 s5, 0xffff, s5
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_and_b32 s1, s6, s2
-; GFX6-NEXT: s_or_b32 s0, s1, s0
-; GFX6-NEXT: s_and_b32 s1, s5, s3
-; GFX6-NEXT: s_mov_b32 s2, s4
-; GFX6-NEXT: s_mov_b32 s3, s5
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_mov_b32 s0, -1
+; GFX6-NEXT: s_mov_b32 s1, 0xffff
+; GFX6-NEXT: s_or_b32 s4, s4, s7
+; GFX6-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX6-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX6-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s6, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s3, 0xffff
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s7, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_or_b32 s2, s2, s3
+; GFX6-NEXT: s_and_b32 s3, s5, 0xffff
; GFX6-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_andn2_v3i16_multi_use:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s0, s4, 16
-; GFX9-NEXT: s_not_b32 s1, s0
-; GFX9-NEXT: s_not_b32 s6, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX9-NEXT: s_lshr_b32 s1, s2, 16
-; GFX9-NEXT: s_not_b32 s7, s5
-; GFX9-NEXT: s_andn2_b32 s0, s1, s0
-; GFX9-NEXT: s_andn2_b32 s1, s2, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX9-NEXT: s_andn2_b32 s1, s3, s5
-; GFX9-NEXT: s_mov_b32 s2, s6
-; GFX9-NEXT: s_mov_b32 s3, s7
+; GFX9-NEXT: s_mov_b64 s[0:1], -1
+; GFX9-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX9-NEXT: s_and_b64 s[0:1], s[2:3], s[4:5]
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s6, s4, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX9-NEXT: s_lshl_b32 s3, s6, 16
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX9-NEXT: s_or_b32 s2, s2, s3
+; GFX9-NEXT: s_and_b32 s3, s5, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_andn2_v3i16_multi_use:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s0, s4, 16
-; GFX10-NEXT: s_not_b32 s6, s4
-; GFX10-NEXT: s_not_b32 s1, s0
-; GFX10-NEXT: s_lshr_b32 s7, s2, 16
-; GFX10-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX10-NEXT: s_andn2_b32 s0, s7, s0
-; GFX10-NEXT: s_andn2_b32 s1, s2, s4
-; GFX10-NEXT: s_not_b32 s4, s5
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX10-NEXT: s_andn2_b32 s1, s3, s5
-; GFX10-NEXT: s_mov_b32 s2, s6
-; GFX10-NEXT: s_mov_b32 s3, s4
+; GFX10-NEXT: s_mov_b64 s[0:1], -1
+; GFX10-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX10-NEXT: s_and_b64 s[0:1], s[2:3], s[4:5]
+; GFX10-NEXT: s_lshr_b32 s3, s4, 16
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_lshl_b32 s3, s3, 16
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT: s_or_b32 s2, s2, s3
+; GFX10-NEXT: s_and_b32 s3, s5, 0xffff
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_andn2_v3i16_multi_use:
@@ -863,18 +922,19 @@ define amdgpu_ps { i48, i48 } @s_andn2_v3i16_multi_use(<3 x i16> inreg %src0, <3
;
; GFX11-FAKE16-LABEL: s_andn2_v3i16_multi_use:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s4
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s0, s7, s0
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, s2, s4
-; GFX11-FAKE16-NEXT: s_not_b32 s4, s5
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_and_not1_b32 s1, s3, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, s6
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, s4
+; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-FAKE16-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX11-FAKE16-NEXT: s_and_b64 s[0:1], s[2:3], s[4:5]
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s3
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s5, 0xffff
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%and = and <3 x i16> %src0, %not.src1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-cse-leaves-dead-cast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-cse-leaves-dead-cast.mir
index 9d58ba32d33ff..d40e8c615502f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-cse-leaves-dead-cast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-cse-leaves-dead-cast.mir
@@ -38,9 +38,6 @@ body: |
; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
; CHECK-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR %load, [[C6]](s32)
; CHECK-NEXT: %unmerge3_7:_(s1) = G_TRUNC [[LSHR6]](s32)
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
; CHECK-NEXT: %negone:_(s1) = G_CONSTANT i1 true
; CHECK-NEXT: %and0:_(s1) = G_XOR %unmerge3_0, %negone
; CHECK-NEXT: %and1:_(s1) = G_XOR %unmerge3_1, %negone
@@ -50,34 +47,37 @@ body: |
; CHECK-NEXT: %and5:_(s1) = G_XOR %unmerge3_5, %negone
; CHECK-NEXT: %and6:_(s1) = G_XOR %unmerge3_6, %negone
; CHECK-NEXT: %and7:_(s1) = G_XOR %unmerge3_7, %negone
- ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(s32) = G_SEXT %and0(s1)
- ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT]], [[C10]]
- ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:_(s32) = G_SEXT %and1(s1)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT1]], [[C10]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[SEXT2:%[0-9]+]]:_(s32) = G_SEXT %and2(s1)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT2]], [[C10]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C8]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[SEXT3:%[0-9]+]]:_(s32) = G_SEXT %and3(s1)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT3]], [[C10]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C9]](s32)
+ ; CHECK-NEXT: [[SEXT:%[0-9]+]]:_(i32) = G_SEXT %and0(s1)
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT]], [[C7]]
+ ; CHECK-NEXT: [[SEXT1:%[0-9]+]]:_(i32) = G_SEXT %and1(s1)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT1]], [[C7]]
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C8]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[SEXT2:%[0-9]+]]:_(i32) = G_SEXT %and2(s1)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT2]], [[C7]]
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C9]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[SEXT3:%[0-9]+]]:_(i32) = G_SEXT %and3(s1)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT3]], [[C7]]
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C10]](i32)
; CHECK-NEXT: %merge0:_(s32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[SEXT4:%[0-9]+]]:_(s32) = G_SEXT %and4(s1)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT4]], [[C10]]
- ; CHECK-NEXT: [[SEXT5:%[0-9]+]]:_(s32) = G_SEXT %and5(s1)
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT5]], [[C10]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL3]]
- ; CHECK-NEXT: [[SEXT6:%[0-9]+]]:_(s32) = G_SEXT %and6(s1)
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SEXT6]], [[C10]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C8]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL4]]
- ; CHECK-NEXT: [[SEXT7:%[0-9]+]]:_(s32) = G_SEXT %and7(s1)
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SEXT7]], [[C10]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C9]](s32)
+ ; CHECK-NEXT: [[SEXT4:%[0-9]+]]:_(i32) = G_SEXT %and4(s1)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SEXT4]], [[C7]]
+ ; CHECK-NEXT: [[SEXT5:%[0-9]+]]:_(i32) = G_SEXT %and5(s1)
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SEXT5]], [[C7]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C8]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL3]]
+ ; CHECK-NEXT: [[SEXT6:%[0-9]+]]:_(i32) = G_SEXT %and6(s1)
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[SEXT6]], [[C7]]
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C9]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL4]]
+ ; CHECK-NEXT: [[SEXT7:%[0-9]+]]:_(i32) = G_SEXT %and7(s1)
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[SEXT7]], [[C7]]
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C10]](i32)
; CHECK-NEXT: %merge1:_(s32) = G_OR [[OR3]], [[SHL5]]
; CHECK-NEXT: %bv:_(<2 x s32>) = G_BUILD_VECTOR %merge0(s32), %merge1(s32)
; CHECK-NEXT: %null:_(p1) = G_CONSTANT i64 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir
index e60627729e6be..f406e1fd876b7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-sext.mir
@@ -60,11 +60,11 @@ body: |
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<2 x s32>)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV]], 8
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV1]], 8
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir
index f10314d889cbe..9f0ef53b922a0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-unmerge-values.mir
@@ -88,15 +88,14 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV3]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[ICMP1]](s1)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT1]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ANYEXT]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ANYEXT1]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG]](s32), [[SEXT_INREG1]](s32)
@@ -130,17 +129,16 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV3]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[ICMP1]](s1)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT1]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ANYEXT]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ANYEXT1]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C2]]
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -172,15 +170,14 @@ body: |
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[UV1]](s32), [[UV3]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s16) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s16) = G_ANYEXT [[ICMP1]](s1)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ANYEXT1]](s16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ANYEXT]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ANYEXT1]](s16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](i32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s32>) = COPY $vgpr0_vgpr1
@@ -645,18 +642,18 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
; CHECK-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY]], 8
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 8
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BITCAST]](<2 x s16>), implicit [[BITCAST1]](<2 x s16>)
@@ -698,30 +695,30 @@ body: |
; CHECK-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY1]], 8
; CHECK-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY2]], 8
; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY3]], 8
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY4]], 8
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG4]], [[C]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG5]], [[C]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG4]], [[C]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG5]], [[C]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG6]], [[C]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG7]], [[C]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG6]], [[C]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG7]], [[C]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BITCAST]](<2 x s16>), implicit [[BITCAST1]](<2 x s16>), implicit [[BITCAST2]](<2 x s16>), implicit [[BITCAST3]](<2 x s16>)
@@ -783,26 +780,26 @@ body: |
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY5]], 8
; CHECK-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY6]], 8
; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY7]], 8
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG4]], [[C]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG5]], [[C]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG4]], [[C]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG5]], [[C]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG6]], [[C]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG7]], [[C]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG6]], [[C]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG7]], [[C]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY8]], 8
@@ -813,24 +810,24 @@ body: |
; CHECK-NEXT: [[SEXT_INREG13:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY13]], 8
; CHECK-NEXT: [[SEXT_INREG14:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY14]], 8
; CHECK-NEXT: [[SEXT_INREG15:%[0-9]+]]:_(s32) = G_SEXT_INREG [[COPY15]], 8
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG8]], [[C]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG9]], [[C]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG8]], [[C]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG9]], [[C]]
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C1]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL4]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG10]], [[C]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG11]], [[C]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG10]], [[C]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG11]], [[C]]
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C1]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL5]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG12]], [[C]]
- ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG13]], [[C]]
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG12]], [[C]]
+ ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG13]], [[C]]
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND13]], [[C1]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND12]], [[SHL6]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG14]], [[C]]
- ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG15]], [[C]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG14]], [[C]]
+ ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG15]], [[C]]
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND15]], [[C1]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND14]], [[SHL7]]
; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BITCAST]](<2 x s16>), implicit [[BITCAST1]](<2 x s16>), implicit [[BITCAST2]](<2 x s16>), implicit [[BITCAST3]](<2 x s16>), implicit [[BITCAST4]](<2 x s16>), implicit [[BITCAST5]](<2 x s16>), implicit [[BITCAST6]](<2 x s16>), implicit [[BITCAST7]](<2 x s16>)
@@ -1302,21 +1299,21 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<4 x s64>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[UV]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[UV1]](s64)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[UV]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i32) = G_TRUNC [[UV1]](s64)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s64), [[UV5:%[0-9]+]]:_(s64), [[UV6:%[0-9]+]]:_(s64), [[UV7:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<4 x s64>)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[UV6]](s64)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C]]
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[UV7]](s64)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(i32) = G_TRUNC [[UV6]](s64)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[TRUNC2]], [[C]]
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i32) = G_TRUNC [[UV7]](s64)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[TRUNC3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BITCAST]](<2 x s16>), implicit [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir
index 5956d40b6b825..b3cbb2a86d31c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/artifact-combiner-zext.mir
@@ -56,10 +56,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s32>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(<2 x s16>) = G_TRUNC [[COPY]](<2 x s32>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[C]](i32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[AND:%[0-9]+]]:_(<2 x s16>) = G_AND [[TRUNC]], [[BITCAST]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
index eb91938582fb6..e7e161060f4af 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ashr.ll
@@ -806,22 +806,24 @@ define <2 x i16> @v_ashr_v2i16(<2 x i16> %value, <2 x i16> %amount) {
; GFX6-LABEL: v_ashr_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, v3, v2
-; GFX6-NEXT: v_ashrrev_i32_e32 v0, v1, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX6-NEXT: v_bfe_i32 v3, v0, 0, 16
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, v2, v0
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, v1, v3
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ashr_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ashrrev_i16_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, v1, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v2, v1, v0
+; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ashr_v2i16:
@@ -844,9 +846,12 @@ define <2 x i16> @v_ashr_v2i16_15(<2 x i16> %value) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_bfe_i32 v1, v0, 0, 16
-; GFX6-NEXT: v_ashrrev_i32_e32 v0, 31, v0
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 15, v0
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 15, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_bfe_u32 v1, v1, 15, 16
; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
@@ -878,25 +883,28 @@ define amdgpu_ps i32 @s_ashr_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amou
; GFX6-LABEL: s_ashr_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_lshr_b32 s2, s1, 16
-; GFX6-NEXT: s_ashr_i32 s3, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: s_ashr_i32 s2, s3, s2
-; GFX6-NEXT: s_ashr_i32 s0, s0, s1
-; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_sext_i32_i16 s3, s0
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100010
+; GFX6-NEXT: s_ashr_i32 s0, s0, s2
+; GFX6-NEXT: s_ashr_i32 s1, s3, s1
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_or_b32 s0, s1, s0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ashr_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_ashr_i32 s3, s0, 16
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_ashr_i32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_ashr_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_sext_i32_i16 s1, s2
+; GFX8-NEXT: s_ashr_i32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ashr_v2i16:
@@ -928,23 +936,25 @@ define amdgpu_ps i32 @s_ashr_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amou
define amdgpu_ps float @ashr_v2i16_sv(<2 x i16> inreg %value, <2 x i16> %amount) {
; GFX6-LABEL: ashr_v2i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s1, s0, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_ashr_i32_e32 v1, s1, v1
-; GFX6-NEXT: v_ashr_i32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_sext_i32_i16 s1, s0
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100010
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_ashr_i32_e32 v1, s0, v1
+; GFX6-NEXT: v_ashr_i32_e32 v0, s1, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ashr_v2i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_ashrrev_i16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_ashrrev_i16_e64 v0, v0, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_ashrrev_i16_e64 v1, v0, s0
+; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ashr_v2i16_sv:
@@ -964,23 +974,25 @@ define amdgpu_ps float @ashr_v2i16_sv(<2 x i16> inreg %value, <2 x i16> %amount)
define amdgpu_ps float @ashr_v2i16_vs(<2 x i16> %value, <2 x i16> inreg %amount) {
; GFX6-LABEL: ashr_v2i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX6-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, s1, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, s1, v0
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, s0, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ashr_v2i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_ashrrev_i16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, s0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_ashrrev_i16_e32 v1, s0, v0
+; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ashr_v2i16_vs:
@@ -1012,33 +1024,37 @@ define <2 x float> @v_ashr_v4i16(<4 x i16> %value, <4 x i16> %amount) {
; GFX6-LABEL: v_ashr_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_bfe_i32 v6, v0, 0, 16
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_ashrrev_i32_e32 v6, v7, v6
-; GFX6-NEXT: v_ashrrev_i32_e32 v0, v2, v0
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, v5, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, v3, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, v4, v0
+; GFX6-NEXT: v_bfe_i32 v4, v1, 0, 16
+; GFX6-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, v2, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, v5, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, v3, v4
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v6
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ashr_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ashrrev_i16_sdwa v4, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, v2, v0
-; GFX8-NEXT: v_ashrrev_i16_sdwa v2, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, v3, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v4, v2, v0
+; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_ashrrev_i16_e32 v2, v3, v1
+; GFX8-NEXT: v_ashrrev_i16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ashr_v4i16:
@@ -1063,41 +1079,47 @@ define amdgpu_ps <2 x i32> @s_ashr_v4i16(<4 x i16> inreg %value, <4 x i16> inreg
; GFX6-LABEL: s_ashr_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_lshr_b32 s4, s2, 16
-; GFX6-NEXT: s_ashr_i32 s5, s0, 16
-; GFX6-NEXT: s_lshr_b32 s6, s3, 16
-; GFX6-NEXT: s_ashr_i32 s7, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_ashr_i32 s6, s7, s6
-; GFX6-NEXT: s_ashr_i32 s1, s1, s3
-; GFX6-NEXT: s_ashr_i32 s3, s5, s4
-; GFX6-NEXT: s_ashr_i32 s0, s0, s2
-; GFX6-NEXT: s_lshl_b32 s6, s6, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_sext_i32_i16 s6, s0
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100010
+; GFX6-NEXT: s_lshr_b32 s5, s3, 16
+; GFX6-NEXT: s_ashr_i32 s0, s0, s4
+; GFX6-NEXT: s_sext_i32_i16 s4, s1
+; GFX6-NEXT: s_bfe_i32 s1, s1, 0x100010
+; GFX6-NEXT: s_ashr_i32 s2, s6, s2
+; GFX6-NEXT: s_ashr_i32 s1, s1, s5
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s1, s1, s6
-; GFX6-NEXT: s_or_b32 s0, s0, s3
+; GFX6-NEXT: s_ashr_i32 s3, s4, s3
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_or_b32 s0, s2, s0
+; GFX6-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s1, s2, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ashr_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_ashr_i32 s5, s1, 16
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_ashr_i32 s4, s5, s4
-; GFX8-NEXT: s_ashr_i32 s1, s1, s3
-; GFX8-NEXT: s_lshl_b32 s4, s4, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s1, s4
-; GFX8-NEXT: s_lshr_b32 s3, s2, 16
-; GFX8-NEXT: s_ashr_i32 s4, s0, 16
+; GFX8-NEXT: s_lshr_b32 s4, s0, 16
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_ashr_i32 s3, s4, s3
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
; GFX8-NEXT: s_ashr_i32 s0, s0, s2
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_sext_i32_i16 s2, s4
+; GFX8-NEXT: s_lshr_b32 s5, s1, 16
+; GFX8-NEXT: s_ashr_i32 s2, s2, s6
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
+; GFX8-NEXT: s_ashr_i32 s1, s1, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s5
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_ashr_i32 s3, s3, s7
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ashr_v4i16:
@@ -1167,55 +1189,63 @@ define <4 x float> @v_ashr_v8i16(<8 x i16> %value, <8 x i16> %amount) {
; GFX6-LABEL: v_ashr_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v12, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v14, 16, v0
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v0, v4, v0
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, v13, v12
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, v5, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v10, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX6-NEXT: v_bfe_i32 v12, v0, 0, 16
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, v8, v0
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX6-NEXT: v_bfe_i32 v8, v1, 0, 16
+; GFX6-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; GFX6-NEXT: v_ashrrev_i32_e32 v4, v4, v12
+; GFX6-NEXT: v_ashrrev_i32_e32 v5, v5, v8
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, v9, v1
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX6-NEXT: v_bfe_i32 v8, v2, 0, 16
+; GFX6-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; GFX6-NEXT: v_ashrrev_i32_e32 v6, v6, v8
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, v10, v2
+; GFX6-NEXT: v_bfe_i32 v8, v3, 0, 16
+; GFX6-NEXT: v_bfe_i32 v3, v3, 16, 16
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, v11, v10
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, v6, v2
-; GFX6-NEXT: v_ashrrev_i32_e32 v8, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, v11, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_ashrrev_i32_e32 v14, v15, v14
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, v9, v8
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, v7, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v7, v7, v8
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v14
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ashr_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ashrrev_i16_sdwa v8, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, v4, v0
-; GFX8-NEXT: v_ashrrev_i16_sdwa v4, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: v_ashrrev_i16_sdwa v4, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_ashrrev_i16_sdwa v4, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v8, v4, v0
+; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_ashrrev_i16_e32 v4, v5, v1
+; GFX8-NEXT: v_ashrrev_i16_sdwa v1, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_ashrrev_i16_e32 v4, v6, v2
+; GFX8-NEXT: v_ashrrev_i16_sdwa v2, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_ashrrev_i16_e32 v4, v7, v3
+; GFX8-NEXT: v_ashrrev_i16_sdwa v3, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ashr_v8i16:
@@ -1244,73 +1274,85 @@ define amdgpu_ps <4 x i32> @s_ashr_v8i16(<8 x i16> inreg %value, <8 x i16> inreg
; GFX6-LABEL: s_ashr_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_lshr_b32 s8, s4, 16
-; GFX6-NEXT: s_ashr_i32 s9, s0, 16
-; GFX6-NEXT: s_lshr_b32 s10, s5, 16
-; GFX6-NEXT: s_ashr_i32 s11, s1, 16
-; GFX6-NEXT: s_lshr_b32 s12, s6, 16
-; GFX6-NEXT: s_ashr_i32 s13, s2, 16
-; GFX6-NEXT: s_lshr_b32 s14, s7, 16
-; GFX6-NEXT: s_ashr_i32 s15, s3, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_ashr_i32 s14, s15, s14
-; GFX6-NEXT: s_ashr_i32 s3, s3, s7
-; GFX6-NEXT: s_ashr_i32 s7, s13, s12
-; GFX6-NEXT: s_ashr_i32 s2, s2, s6
-; GFX6-NEXT: s_ashr_i32 s6, s11, s10
-; GFX6-NEXT: s_ashr_i32 s1, s1, s5
-; GFX6-NEXT: s_ashr_i32 s5, s9, s8
-; GFX6-NEXT: s_ashr_i32 s0, s0, s4
-; GFX6-NEXT: s_lshl_b32 s14, s14, 16
-; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX6-NEXT: s_lshl_b32 s7, s7, 16
-; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX6-NEXT: s_lshl_b32 s6, s6, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s5, s5, 16
+; GFX6-NEXT: s_sext_i32_i16 s12, s0
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100010
+; GFX6-NEXT: s_lshr_b32 s9, s5, 16
+; GFX6-NEXT: s_ashr_i32 s0, s0, s8
+; GFX6-NEXT: s_sext_i32_i16 s8, s1
+; GFX6-NEXT: s_bfe_i32 s1, s1, 0x100010
+; GFX6-NEXT: s_lshr_b32 s10, s6, 16
+; GFX6-NEXT: s_ashr_i32 s4, s12, s4
+; GFX6-NEXT: s_ashr_i32 s5, s8, s5
+; GFX6-NEXT: s_ashr_i32 s1, s1, s9
+; GFX6-NEXT: s_sext_i32_i16 s8, s2
+; GFX6-NEXT: s_bfe_i32 s2, s2, 0x100010
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s3, s3, s14
-; GFX6-NEXT: s_or_b32 s2, s2, s7
-; GFX6-NEXT: s_or_b32 s1, s1, s6
-; GFX6-NEXT: s_or_b32 s0, s0, s5
+; GFX6-NEXT: s_lshr_b32 s11, s7, 16
+; GFX6-NEXT: s_ashr_i32 s6, s8, s6
+; GFX6-NEXT: s_ashr_i32 s2, s2, s10
+; GFX6-NEXT: s_sext_i32_i16 s8, s3
+; GFX6-NEXT: s_bfe_i32 s3, s3, 0x100010
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_ashr_i32 s3, s3, s11
+; GFX6-NEXT: s_or_b32 s0, s4, s0
+; GFX6-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_ashr_i32 s7, s8, s7
+; GFX6-NEXT: s_or_b32 s1, s4, s1
+; GFX6-NEXT: s_and_b32 s4, s6, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_or_b32 s2, s4, s2
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s3, s4, s3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ashr_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s8, s7, 16
-; GFX8-NEXT: s_ashr_i32 s9, s3, 16
-; GFX8-NEXT: s_sext_i32_i16 s3, s3
-; GFX8-NEXT: s_ashr_i32 s8, s9, s8
-; GFX8-NEXT: s_ashr_i32 s3, s3, s7
-; GFX8-NEXT: s_lshl_b32 s8, s8, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_or_b32 s3, s3, s8
-; GFX8-NEXT: s_lshr_b32 s7, s6, 16
-; GFX8-NEXT: s_ashr_i32 s8, s2, 16
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_ashr_i32 s7, s8, s7
-; GFX8-NEXT: s_ashr_i32 s2, s2, s6
-; GFX8-NEXT: s_lshl_b32 s7, s7, 16
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s2, s2, s7
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_ashr_i32 s7, s1, 16
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_ashr_i32 s6, s7, s6
-; GFX8-NEXT: s_ashr_i32 s1, s1, s5
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s1, s6
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_ashr_i32 s6, s0, 16
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_ashr_i32 s5, s6, s5
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
; GFX8-NEXT: s_ashr_i32 s0, s0, s4
-; GFX8-NEXT: s_lshl_b32 s5, s5, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s5
+; GFX8-NEXT: s_sext_i32_i16 s4, s8
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: s_ashr_i32 s4, s4, s12
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
+; GFX8-NEXT: s_ashr_i32 s1, s1, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s9
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: s_ashr_i32 s5, s5, s13
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: s_ashr_i32 s2, s2, s6
+; GFX8-NEXT: s_sext_i32_i16 s6, s10
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s5
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
+; GFX8-NEXT: s_ashr_i32 s6, s6, s14
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: s_ashr_i32 s3, s3, s7
+; GFX8-NEXT: s_sext_i32_i16 s7, s11
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_ashr_i32 s7, s7, s15
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ashr_v8i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll
index 24ecf293fa6de..5738dd30112c5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/bswap.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-amd-amdpal -o - %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal -o - %s | FileCheck -check-prefix=GFX7 %s
; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefix=GFX10 %s
@@ -447,13 +447,17 @@ define i16 @v_bswap_i16(i16 %src) {
define amdgpu_ps i32 @s_bswap_v2i16(<2 x i16> inreg %src) {
; GFX7-LABEL: s_bswap_v2i16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: v_alignbit_b32 v0, s0, s0, 8
-; GFX7-NEXT: v_alignbit_b32 v1, s0, s0, 24
-; GFX7-NEXT: s_mov_b32 s0, 0xff00ff
-; GFX7-NEXT: v_bfi_b32 v0, s0, v1, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, v0
-; GFX7-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
+; GFX7-NEXT: s_lshr_b32 s1, s0, 16
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: v_alignbit_b32 v0, s1, v0, 24
+; GFX7-NEXT: s_lshl_b32 s2, s0, 8
+; GFX7-NEXT: s_bfe_u32 s3, s0, 0x80008
; GFX7-NEXT: v_readfirstlane_b32 s0, v0
+; GFX7-NEXT: s_or_b32 s2, s3, s2
+; GFX7-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX7-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX7-NEXT: s_lshl_b32 s0, s0, 16
+; GFX7-NEXT: s_or_b32 s0, s1, s0
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_bswap_v2i16:
@@ -557,11 +561,15 @@ define <2 x i16> @v_bswap_v2i16(<2 x i16> %src) {
; GFX7-LABEL: v_bswap_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_alignbit_b32 v1, v0, v0, 8
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v0, 24
-; GFX7-NEXT: s_mov_b32 s4, 0xff00ff
-; GFX7-NEXT: v_bfi_b32 v0, s4, v0, v1
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v0, 16
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v0
+; GFX7-NEXT: v_bfe_u32 v3, v0, 8, 8
+; GFX7-NEXT: v_alignbit_b32 v0, v1, v0, 24
+; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_bswap_v2i16:
@@ -591,15 +599,19 @@ define <3 x i16> @v_bswap_v3i16(<3 x i16> %src) {
; GFX7-LABEL: v_bswap_v3i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_alignbit_b32 v2, v0, v0, 8
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v0, 24
-; GFX7-NEXT: s_mov_b32 s4, 0xff00ff
-; GFX7-NEXT: v_bfi_b32 v0, s4, v0, v2
-; GFX7-NEXT: v_alignbit_b32 v2, v1, v1, 8
-; GFX7-NEXT: v_alignbit_b32 v1, v1, v1, 24
-; GFX7-NEXT: v_bfi_b32 v1, s4, v1, v2
-; GFX7-NEXT: v_alignbit_b32 v0, v0, v0, 16
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v0
+; GFX7-NEXT: v_bfe_u32 v4, v0, 8, 8
+; GFX7-NEXT: v_alignbit_b32 v0, v2, v0, 24
+; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v1
+; GFX7-NEXT: v_bfe_u32 v1, v1, 8, 8
+; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_bswap_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
index 638476f965aa2..cb727f183556b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/cvt_f32_ubyte.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-- < %s | FileCheck -check-prefixes=GCN,SI %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-- < %s | FileCheck -check-prefixes=GCN,VI %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-- < %s | FileCheck -check-prefixes=GCN,SI %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-- < %s | FileCheck -check-prefixes=GCN,VI %s
declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
@@ -726,81 +726,87 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %o
; SI-LABEL: load_v4i8_to_v4f32_2_uses:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: s_mov_b32 s3, s7
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v4, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_mov_b32 s10, s6
-; SI-NEXT: s_mov_b32 s11, s7
+; SI-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
-; SI-NEXT: s_mov_b32 s8, s2
-; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: s_mov_b64 s[0:1], s[4:5]
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; SI-NEXT: v_and_b32_e32 v6, 0xff00, v4
-; SI-NEXT: v_cvt_f32_ubyte3_e32 v3, v4
-; SI-NEXT: v_cvt_f32_ubyte2_e32 v2, v4
-; SI-NEXT: v_cvt_f32_ubyte1_e32 v1, v4
-; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v4
+; SI-NEXT: v_lshrrev_b32_e32 v1, 8, v0
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v0
+; SI-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; SI-NEXT: v_cvt_f32_ubyte3_e32 v3, v0
+; SI-NEXT: v_add_i32_e32 v6, vcc, 9, v0
+; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v5
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v1
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v2
+; SI-NEXT: v_add_i32_e32 v8, vcc, 9, v1
+; SI-NEXT: v_add_i32_e32 v9, vcc, 9, v2
+; SI-NEXT: v_cvt_f32_ubyte0_e32 v1, v5
+; SI-NEXT: v_cvt_f32_ubyte0_e32 v2, v7
+; SI-NEXT: v_and_b32_e32 v5, 0xff, v8
; SI-NEXT: v_add_i32_e32 v4, vcc, 9, v4
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
+; SI-NEXT: v_and_b32_e32 v6, 0xff, v6
+; SI-NEXT: v_and_b32_e32 v7, 0xff, v9
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, 0xff, v4
-; SI-NEXT: v_add_i32_e32 v2, vcc, 9, v5
-; SI-NEXT: v_and_b32_e32 v1, 0xff00, v5
+; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v5
+; SI-NEXT: v_and_b32_e32 v4, 0xff, v4
+; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v7
; SI-NEXT: v_or_b32_e32 v0, v6, v0
-; SI-NEXT: v_and_b32_e32 v2, 0xff, v2
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x900, v0
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: v_add_i32_e32 v0, vcc, 0x9000000, v0
-; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; SI-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-NEXT: s_mov_b64 s[0:1], s[6:7]
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: load_v4i8_to_v4f32_2_uses:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; VI-NEXT: v_mov_b32_e32 v9, 0xffffff00
-; VI-NEXT: v_mov_b32_e32 v10, 9
-; VI-NEXT: v_mov_b32_e32 v11, 0x900
+; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-NEXT: v_mov_b32_e32 v6, 9
+; VI-NEXT: v_mov_b32_e32 v7, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v8, v[0:1]
+; VI-NEXT: flat_load_dword v1, v[0:1]
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0xff
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v6, s2
-; VI-NEXT: v_mov_b32_e32 v7, s3
+; VI-NEXT: v_mov_b32_e32 v4, s0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cvt_f32_ubyte3_e32 v3, v8
-; VI-NEXT: v_cvt_f32_ubyte2_e32 v2, v8
-; VI-NEXT: v_cvt_f32_ubyte1_e32 v1, v8
-; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v8
-; VI-NEXT: v_and_b32_e32 v12, 0xffffff00, v8
-; VI-NEXT: v_add_u16_e32 v13, 9, v8
-; VI-NEXT: v_and_b32_sdwa v9, v8, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_add_u16_sdwa v8, v8, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_lshrrev_b32_e32 v8, 8, v1
+; VI-NEXT: v_and_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_cvt_f32_ubyte0_sdwa v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT: v_cvt_f32_ubyte3_e32 v3, v1
+; VI-NEXT: v_add_u16_e32 v9, 9, v1
+; VI-NEXT: v_add_u16_sdwa v10, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-NEXT: v_add_u16_sdwa v6, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; VI-NEXT: v_cvt_f32_ubyte0_sdwa v1, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0
+; VI-NEXT: v_cvt_f32_ubyte0_e32 v2, v2
+; VI-NEXT: v_add_u16_e32 v8, 9, v8
+; VI-NEXT: v_and_b32_e32 v10, 0xff, v10
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: s_nop 0
-; VI-NEXT: v_or_b32_sdwa v0, v12, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT: v_or_b32_sdwa v1, v9, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; VI-NEXT: v_add_u16_e32 v0, 0x900, v0
-; VI-NEXT: v_add_u16_sdwa v1, v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_and_b32_e32 v6, 0xff, v6
+; VI-NEXT: v_lshlrev_b32_sdwa v0, v7, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v10
+; VI-NEXT: v_or_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; VI-NEXT: v_lshlrev_b32_e32 v2, 24, v6
; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: flat_store_dword v[6:7], v0
+; VI-NEXT: v_or_b32_e32 v2, v0, v2
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%tid.x = call i32 @llvm.amdgcn.workitem.id.x()
%in.ptr = getelementptr <4 x i8>, ptr addrspace(1) %in, i32 %tid.x
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
index f5308915e6065..aa389eec4f991 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dummy-target.ll
@@ -35,10 +35,9 @@ define <2 x i16> @vop3p_add_v2i16(<2 x i16> %arg0) #0 {
; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR1]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i16) = G_ADD [[TRUNC]], [[TRUNC2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i16) = G_ADD [[TRUNC1]], [[TRUNC3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](i16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](i16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](i16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](i16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x i16>)
@@ -74,11 +73,10 @@ define <2 x i16> @halfinsts_add_v2i16(<2 x i16> %arg0) #1 {
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(i32) = G_ADD [[BITCAST1]], [[BITCAST2]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(i32) = G_ADD [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST3]](<2 x i16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
index 9a28dea79e3ff..32c30f76484e9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fdiv.f16.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; Denormal mode shouldn't matter for f16, check with and without flushing.
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX6,GFX6-IEEE %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX6,GFX6-FLUSH %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX6,GFX6-IEEE %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX6,GFX6-FLUSH %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-IEEE %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-FLUSH %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-IEEE %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX89,GFX8,GFX8-FLUSH %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -denormal-fp-math=ieee < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-IEEE %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 -denormal-fp-math=preserve-sign < %s | FileCheck -check-prefixes=GFX89,GFX9,GFX9-FLUSH %s
@@ -744,47 +744,49 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX6-IEEE-LABEL: v_fdiv_v2f16:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v2
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v3, v2, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-IEEE-NEXT: v_fma_f32 v7, -v4, v5, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v5, v5
; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v6, v5
; GFX6-IEEE-NEXT: v_fma_f32 v8, -v4, v7, v6
; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v5, v7
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v4, v7, v6
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[4:5], v1, v1, v0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v8, v6
; GFX6-IEEE-NEXT: v_div_fmas_f32 v4, v4, v5, v7
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v2, v4, v2, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v6, v8, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v8, v8
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v6, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v4, -v6, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v4, v3, v5
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v2, v4, v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, v0, v1, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v4, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v5, v6, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, v3
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v4, v6
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v3, v1, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_fdiv_v2f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v2
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v5, v4
; GFX6-FLUSH-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
@@ -796,8 +798,8 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX6-FLUSH-NEXT: v_fma_f32 v7, v8, v5, v7
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v4, v7, v6
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v4, v4, v5, v7
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v2, v4, v3, v2
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -816,42 +818,85 @@ define <2 x half> @v_fdiv_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v3, v3, v4, v6
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v3, v1, v0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_fdiv_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX8-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v5, v4
-; GFX8-NEXT: v_rcp_f32_e32 v7, v3
-; GFX8-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX8-NEXT: v_rcp_f32_e32 v9, v6
-; GFX8-NEXT: v_mul_f32_e32 v10, v5, v7
-; GFX8-NEXT: v_mad_f32 v11, -v3, v10, v5
-; GFX8-NEXT: v_mac_f32_e32 v10, v11, v7
-; GFX8-NEXT: v_mad_f32 v3, -v3, v10, v5
-; GFX8-NEXT: v_mul_f32_e32 v5, v8, v9
-; GFX8-NEXT: v_mul_f32_e32 v3, v3, v7
-; GFX8-NEXT: v_mad_f32 v7, -v6, v5, v8
-; GFX8-NEXT: v_mac_f32_e32 v5, v7, v9
-; GFX8-NEXT: v_mad_f32 v6, -v6, v5, v8
-; GFX8-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX8-NEXT: v_mul_f32_e32 v6, v6, v9
-; GFX8-NEXT: v_add_f32_e32 v3, v3, v10
-; GFX8-NEXT: v_and_b32_e32 v6, 0xff800000, v6
-; GFX8-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GFX8-NEXT: v_div_fixup_f16 v2, v3, v2, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_div_fixup_f16 v0, v5, v1, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_fdiv_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v8, v6
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v2
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v7, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v9, v4, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v10, -v2, v9
+; GFX8-IEEE-NEXT: v_add_f32_e32 v10, v10, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v10, v10, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v9, v10, v9
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v9
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v8
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v7, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v9, -v8, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v9, v9, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v9, v9, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v8, -v8, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, v8, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v4, v7, v4
+; GFX8-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v4, v6, v3
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_fdiv_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v8, v6
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v2
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v7, v3
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v9, v4, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v10, -v2, v9, v4
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v9, v10, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v9, v4
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v8
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v7, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v9, -v8, v5, v7
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v9, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, -v8, v5, v7
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v4, v7, v4
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v6, v3
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_fdiv_v2f16:
; GFX9-IEEE: ; %bb.0:
@@ -1024,30 +1069,34 @@ define <2 x half> @v_fdiv_v2f16_afn(<2 x half> %a, <2 x half> %b) {
; GFX6-LABEL: v_fdiv_v2f16_afn:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_rcp_f32_e32 v1, v1
-; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fdiv_v2f16_afn:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e32 v1, v1
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_rcp_f16_e32 v2, v1
+; GFX8-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fdiv_v2f16_afn:
@@ -1090,47 +1139,49 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-IEEE-LABEL: v_fdiv_v2f16_ulp25:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v2
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v3, v2, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-IEEE-NEXT: v_fma_f32 v7, -v4, v5, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v5, v5
; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v6, v5
; GFX6-IEEE-NEXT: v_fma_f32 v8, -v4, v7, v6
; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v5, v7
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v4, v7, v6
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[4:5], v1, v1, v0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v8, v6
; GFX6-IEEE-NEXT: v_div_fmas_f32 v4, v4, v5, v7
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v2, v4, v2, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v6, v8, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v8, v8
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v6, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v4, -v6, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v4, v3, v5
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v2, v4, v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, v0, v1, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v4, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v5, v6, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, v3
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v4, v6
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v3, v1, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_fdiv_v2f16_ulp25:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v2
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v5, v4
; GFX6-FLUSH-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
@@ -1142,8 +1193,8 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-FLUSH-NEXT: v_fma_f32 v7, v8, v5, v7
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v4, v7, v6
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v4, v4, v5, v7
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v2, v4, v3, v2
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -1162,42 +1213,85 @@ define <2 x half> @v_fdiv_v2f16_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v3, v3, v4, v6
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v3, v1, v0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_fdiv_v2f16_ulp25:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX8-NEXT: v_cvt_f32_f16_e32 v6, v1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v5, v4
-; GFX8-NEXT: v_rcp_f32_e32 v7, v3
-; GFX8-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX8-NEXT: v_rcp_f32_e32 v9, v6
-; GFX8-NEXT: v_mul_f32_e32 v10, v5, v7
-; GFX8-NEXT: v_mad_f32 v11, -v3, v10, v5
-; GFX8-NEXT: v_mac_f32_e32 v10, v11, v7
-; GFX8-NEXT: v_mad_f32 v3, -v3, v10, v5
-; GFX8-NEXT: v_mul_f32_e32 v5, v8, v9
-; GFX8-NEXT: v_mul_f32_e32 v3, v3, v7
-; GFX8-NEXT: v_mad_f32 v7, -v6, v5, v8
-; GFX8-NEXT: v_mac_f32_e32 v5, v7, v9
-; GFX8-NEXT: v_mad_f32 v6, -v6, v5, v8
-; GFX8-NEXT: v_and_b32_e32 v3, 0xff800000, v3
-; GFX8-NEXT: v_mul_f32_e32 v6, v6, v9
-; GFX8-NEXT: v_add_f32_e32 v3, v3, v10
-; GFX8-NEXT: v_and_b32_e32 v6, 0xff800000, v6
-; GFX8-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX8-NEXT: v_add_f32_e32 v5, v6, v5
-; GFX8-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GFX8-NEXT: v_div_fixup_f16 v2, v3, v2, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_div_fixup_f16 v0, v5, v1, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_fdiv_v2f16_ulp25:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v8, v6
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v2
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v7, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v9, v4, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v10, -v2, v9
+; GFX8-IEEE-NEXT: v_add_f32_e32 v10, v10, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v10, v10, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v9, v10, v9
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v9
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v8
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v7, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v9, -v8, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v9, v9, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v9, v9, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v9, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v8, -v8, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, v8, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v4, v7, v4
+; GFX8-IEEE-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v4, v6, v3
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_fdiv_v2f16_ulp25:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v8, v6
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v2
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v7, v3
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v9, v4, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v10, -v2, v9, v4
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v9, v10, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v9, v4
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v8
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v9
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v7, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v9, -v8, v5, v7
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v9, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, -v8, v5, v7
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v4, v7, v4
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v4, 0xff800000, v4
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v4, v4, v5
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v1, v0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v4, v6, v3
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_fdiv_v2f16_ulp25:
; GFX9-IEEE: ; %bb.0:
@@ -1370,42 +1464,44 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX6-IEEE-LABEL: v_rcp_v2f16:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, 1.0, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v3, v5, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v8, v5, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v9, v5, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v8
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v4, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v2, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_rcp_v2f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
@@ -1426,7 +1522,6 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -1437,16 +1532,75 @@ define <2 x half> @v_rcp_v2f16(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_rcp_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_rcp_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, 1.0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_rcp_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v1, v4, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v1, -v1, v6, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_rcp_v2f16:
; GFX9-IEEE: ; %bb.0:
@@ -1603,42 +1757,44 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX6-IEEE-LABEL: v_neg_rcp_v2f16:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, -1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, -1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, -1.0, v1, -1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v3, v5, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v8, v5, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v9, v5, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v8
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, -1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v4, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, -1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, -1.0, v1, -1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, -1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, -1.0, v0, -1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v2, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_neg_rcp_v2f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, -1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v1, -1.0
@@ -1659,7 +1815,6 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -1670,16 +1825,75 @@ define <2 x half> @v_neg_rcp_v2f16(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_neg_rcp_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, -v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e64 v0, -v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_neg_rcp_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v3, v5
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, -1.0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v4, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, -1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_neg_rcp_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v1, v4, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v1, -v1, v6, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_neg_rcp_v2f16:
; GFX9-IEEE: ; %bb.0:
@@ -1836,42 +2050,46 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-IEEE-LABEL: v_rcp_v2f16_fabs:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, 1.0, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v3, v5, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v8, v5, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v9, v5, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v8
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v4, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v2, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_rcp_v2f16_fabs:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; GFX6-FLUSH-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
@@ -1883,7 +2101,7 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -1892,7 +2110,6 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -1903,16 +2120,77 @@ define <2 x half> @v_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_rcp_v2f16_fabs:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, |v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e64 v0, |v0|
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_rcp_v2f16_fabs:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, 1.0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_rcp_v2f16_fabs:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v1, v4, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v1, -v1, v6, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_rcp_v2f16_fabs:
; GFX9-IEEE: ; %bb.0:
@@ -2078,42 +2356,46 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-IEEE-LABEL: v_neg_rcp_v2f16_fabs:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, -1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, -1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, -1.0, v1, -1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v3, v5, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v8, v5, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v9, v5, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v8
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, -1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v4, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, -1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, -1.0, v1, -1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, -1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, -1.0, v0, -1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v2, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_neg_rcp_v2f16_fabs:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; GFX6-FLUSH-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, -1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v1, -1.0
@@ -2125,7 +2407,7 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -2134,7 +2416,6 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -2145,16 +2426,77 @@ define <2 x half> @v_neg_rcp_v2f16_fabs(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_neg_rcp_v2f16_fabs:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, -|v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e64 v0, -|v0|
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_neg_rcp_v2f16_fabs:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v3, v5
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, -1.0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v4, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, -1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_neg_rcp_v2f16_fabs:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v1, v4, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v1, -v1, v6, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, -1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, -1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_neg_rcp_v2f16_fabs:
; GFX9-IEEE: ; %bb.0:
@@ -2320,42 +2662,44 @@ define <2 x half> @v_rcp_v2f16_arcp(<2 x half> %x) {
; GFX6-IEEE-LABEL: v_rcp_v2f16_arcp:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, 1.0, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v3, v5, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v8, v5, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v9, v5, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v8
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v4, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v2, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_rcp_v2f16_arcp:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
@@ -2376,7 +2720,6 @@ define <2 x half> @v_rcp_v2f16_arcp(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -2387,15 +2730,20 @@ define <2 x half> @v_rcp_v2f16_arcp(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_rcp_v2f16_arcp:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX8-NEXT: v_rcp_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_rcp_v2f16_arcp:
@@ -2438,6 +2786,8 @@ define <2 x half> @v_rcp_v2f16_arcp_afn(<2 x half> %x) {
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -2445,9 +2795,11 @@ define <2 x half> @v_rcp_v2f16_arcp_afn(<2 x half> %x) {
; GFX8-LABEL: v_rcp_v2f16_arcp_afn:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX8-NEXT: v_rcp_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_rcp_v2f16_arcp_afn:
@@ -2483,42 +2835,44 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX6-IEEE-LABEL: v_rcp_v2f16_ulp25:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, 1.0, v1, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v3, v5, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v8, v5, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v9, v5, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v8
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v4, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v4
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v4, v2, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v4, v2
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v4
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_rcp_v2f16_ulp25:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v1, 1.0
@@ -2539,7 +2893,6 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -2550,16 +2903,75 @@ define <2 x half> @v_rcp_v2f16_ulp25(<2 x half> %x) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_rcp_v2f16_ulp25:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_rcp_v2f16_ulp25:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v1, -v1, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, 1.0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v4, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v4
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_rcp_v2f16_ulp25:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v1
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v1, v4, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v1, -v1, v6, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v1, v1, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v1, 0xff800000, v1
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v1, v1, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v1, v0, 1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v3, v2, 1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_rcp_v2f16_ulp25:
; GFX9-IEEE: ; %bb.0:
@@ -2716,30 +3128,34 @@ define <2 x half> @v_fdiv_v2f16_afn_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-LABEL: v_fdiv_v2f16_afn_ulp25:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_rcp_f32_e32 v1, v1
-; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fdiv_v2f16_afn_ulp25:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e32 v1, v1
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_rcp_f16_e32 v2, v1
+; GFX8-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fdiv_v2f16_afn_ulp25:
@@ -2782,47 +3198,49 @@ define <2 x half> @v_fdiv_v2f16_arcp_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-IEEE-LABEL: v_fdiv_v2f16_arcp_ulp25:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v2
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v4
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v3, v2, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-IEEE-NEXT: v_fma_f32 v7, -v4, v5, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v5, v5
; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v6, v5
; GFX6-IEEE-NEXT: v_fma_f32 v8, -v4, v7, v6
; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v5, v7
; GFX6-IEEE-NEXT: v_fma_f32 v4, -v4, v7, v6
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[4:5], v1, v1, v0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v8, v6
; GFX6-IEEE-NEXT: v_div_fmas_f32 v4, v4, v5, v7
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v2, v4, v2, v3
-; GFX6-IEEE-NEXT: v_div_scale_f32 v4, vcc, v0, v1, v0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v6, v8, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v3, v8, v8
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v6, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v3, v5
-; GFX6-IEEE-NEXT: v_fma_f32 v4, -v6, v5, v4
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v4, v3, v5
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v1, v1, v0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v2, v4, v3, v2
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, v0, v1, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v4, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v4, v4, v6, v6
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v6, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v5, v6, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, v7, v4, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v6, v3
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v4, v6
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v3, v1, v0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_fdiv_v2f16_arcp_ulp25:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v2
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v5, v4
; GFX6-FLUSH-NEXT: v_div_scale_f32 v6, vcc, v2, v3, v2
@@ -2834,8 +3252,8 @@ define <2 x half> @v_fdiv_v2f16_arcp_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-FLUSH-NEXT: v_fma_f32 v7, v8, v5, v7
; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v4, v7, v6
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v4, v4, v5, v7
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v2, v4, v3, v2
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
@@ -2854,18 +3272,22 @@ define <2 x half> @v_fdiv_v2f16_arcp_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v3, v3, v4, v6
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v3, v1, v0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fdiv_v2f16_arcp_ulp25:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e32 v1, v1
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_rcp_f16_e32 v2, v1
+; GFX8-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fdiv_v2f16_arcp_ulp25:
@@ -2908,30 +3330,34 @@ define <2 x half> @v_fdiv_v2f16_arcp_afn_ulp25(<2 x half> %a, <2 x half> %b) {
; GFX6-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-NEXT: v_rcp_f32_e32 v2, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_rcp_f32_e32 v1, v1
-; GFX6-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_rcp_f32_e32 v3, v3
+; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
+; GFX6-NEXT: v_mul_f32_e32 v0, v0, v3
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f16_sdwa v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rcp_f16_e32 v1, v1
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_rcp_f16_e32 v2, v1
+; GFX8-NEXT: v_rcp_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fdiv_v2f16_arcp_afn_ulp25:
@@ -3275,50 +3701,58 @@ define amdgpu_ps i16 @s_fdiv_f16_afn(i16 inreg %a.arg, i16 inreg %b.arg) {
define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX6-IEEE-LABEL: s_fdiv_v2f16:
; GFX6-IEEE: ; %bb.0:
-; GFX6-IEEE-NEXT: s_lshr_b32 s2, s1, 16
-; GFX6-IEEE-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s3
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v4, s0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v5, s1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s1
+; GFX6-IEEE-NEXT: s_lshr_b32 s4, s0, 16
; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[2:3], v1, v1, v0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, vcc, v0, v1, v0
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v2, v3, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v3, v3
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v7, v6, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v7, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v3, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v7, v6
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[0:1], v5, v5, v4
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v8, v6
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v7
+; GFX6-IEEE-NEXT: s_lshr_b32 s2, s1, 16
+; GFX6-IEEE-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: v_fma_f32 v5, -v2, v3, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v5, v3, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v4, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v6, -v2, v5, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v5, v6, v3, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v5, v4
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v4, s4
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v6, s2
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v1, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, vcc, v4, v5, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v1, -v6, v8, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v1, v1, v8, v8
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v3, v2, v1
-; GFX6-IEEE-NEXT: v_fma_f32 v7, -v6, v3, v2
-; GFX6-IEEE-NEXT: v_fma_f32 v3, v7, v1, v3
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v6, v3, v2
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v1, v2, v1, v3
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v1, v5, v4
+; GFX6-IEEE-NEXT: v_div_scale_f32 v1, s[0:1], v6, v6, v4
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v2, v1
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], v4, v6, v4
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-IEEE-NEXT: v_fma_f32 v5, -v1, v2, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v2, v5, v2, v2
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v5, v3, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v7, -v1, v5, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v5, v7, v2, v5
+; GFX6-IEEE-NEXT: v_fma_f32 v1, -v1, v5, v3
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v1, v1, v2, v5
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v1, v6, v4
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-IEEE-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-IEEE-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX6-IEEE-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX6-IEEE-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-IEEE-NEXT: s_or_b32 s0, s0, s1
; GFX6-IEEE-NEXT: ; return to shader part epilog
;
; GFX6-FLUSH-LABEL: s_fdiv_v2f16:
; GFX6-FLUSH: ; %bb.0:
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-FLUSH-NEXT: s_lshr_b32 s2, s0, 16
; GFX6-FLUSH-NEXT: s_lshr_b32 s3, s1, 16
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s3
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[2:3], v0, v0, v1
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, v0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], v0, v1, v0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, v1, v0, v1
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
@@ -3327,65 +3761,122 @@ define amdgpu_ps i32 @s_fdiv_v2f16(i32 inreg %a.arg, i32 inreg %b.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, s1
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v6, s0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, v1
+; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v1, v0
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s2
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, s3
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v1, s[0:1], v4, v4, v6
-; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v2, v1
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, vcc, v6, v4, v6
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v3, s[0:1], v2, v2, v1
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], v1, v2, v1
+; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
+; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v0, v3
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v1, v2, 1.0
-; GFX6-FLUSH-NEXT: v_fma_f32 v2, v5, v2, v2
-; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v3, v2
-; GFX6-FLUSH-NEXT: v_fma_f32 v7, -v1, v5, v3
-; GFX6-FLUSH-NEXT: v_fma_f32 v5, v7, v2, v5
-; GFX6-FLUSH-NEXT: v_fma_f32 v1, -v1, v5, v3
+; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v3, v0, 1.0
+; GFX6-FLUSH-NEXT: v_fma_f32 v0, v5, v0, v0
+; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v0
+; GFX6-FLUSH-NEXT: v_fma_f32 v6, -v3, v5, v4
+; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v0, v5
+; GFX6-FLUSH-NEXT: v_fma_f32 v3, -v3, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_div_fmas_f32 v1, v1, v2, v5
-; GFX6-FLUSH-NEXT: v_div_fixup_f32 v1, v1, v4, v6
-; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-FLUSH-NEXT: v_div_fmas_f32 v0, v3, v0, v5
+; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v0, v2, v1
+; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT: s_bfe_u32 s1, s2, 0x100000
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-FLUSH-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX6-FLUSH-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-FLUSH-NEXT: s_or_b32 s0, s1, s0
; GFX6-FLUSH-NEXT: ; return to shader part epilog
;
-; GFX8-LABEL: s_fdiv_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, s3
-; GFX8-NEXT: v_cvt_f32_f16_e32 v2, s1
-; GFX8-NEXT: v_rcp_f32_e32 v3, v0
-; GFX8-NEXT: v_cvt_f32_f16_e32 v4, s0
-; GFX8-NEXT: v_rcp_f32_e32 v5, v2
-; GFX8-NEXT: v_mul_f32_e32 v6, v1, v3
-; GFX8-NEXT: v_mad_f32 v7, -v0, v6, v1
-; GFX8-NEXT: v_mac_f32_e32 v6, v7, v3
-; GFX8-NEXT: v_mad_f32 v0, -v0, v6, v1
-; GFX8-NEXT: v_mul_f32_e32 v0, v0, v3
-; GFX8-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX8-NEXT: v_mul_f32_e32 v3, v4, v5
-; GFX8-NEXT: v_add_f32_e32 v0, v0, v6
-; GFX8-NEXT: v_mad_f32 v6, -v2, v3, v4
-; GFX8-NEXT: v_mac_f32_e32 v3, v6, v5
-; GFX8-NEXT: v_mad_f32 v2, -v2, v3, v4
-; GFX8-NEXT: v_mul_f32_e32 v2, v2, v5
-; GFX8-NEXT: v_and_b32_e32 v2, 0xff800000, v2
-; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: v_add_f32_e32 v2, v2, v3
-; GFX8-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_div_fixup_f16 v0, v0, s2, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_div_fixup_f16 v1, v2, s1, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-IEEE-LABEL: s_fdiv_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
+; GFX8-IEEE-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v4, s3
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v2, v0
+; GFX8-IEEE-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v1, v2
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v0, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v0, -v0, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v1, v4
+; GFX8-IEEE-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, s1, v5
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v5, v0, v1
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v4, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v0
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v1
+; GFX8-IEEE-NEXT: v_add_f32_e32 v5, v6, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v4, -v4, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-IEEE-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX8-IEEE-NEXT: v_add_f32_e32 v0, s1, v5
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v2, v3, s0
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s0, v1
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v0, v1, s2
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-IEEE-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-IEEE-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-IEEE-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-IEEE-NEXT: s_or_b32 s0, s0, s1
+; GFX8-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX8-FLUSH-LABEL: s_fdiv_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
+; GFX8-FLUSH-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v4, s3
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v2, v0
+; GFX8-FLUSH-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v1, v2
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v0, v5, v1
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v2
+; GFX8-FLUSH-NEXT: v_mad_f32 v0, -v0, v5, v1
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v2
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v1, v4
+; GFX8-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, s1, v5
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v5, v0, v1
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v4, v5, v0
+; GFX8-FLUSH-NEXT: v_mac_f32_e32 v5, v6, v1
+; GFX8-FLUSH-NEXT: v_mad_f32 v0, -v4, v5, v0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-FLUSH-NEXT: s_and_b32 s1, s1, 0xff800000
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v0, s1, v5
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v3, s0
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s0, v1
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v0, v1, s2
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-FLUSH-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-FLUSH-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-FLUSH-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-FLUSH-NEXT: s_or_b32 s0, s0, s1
+; GFX8-FLUSH-NEXT: ; return to shader part epilog
;
; GFX9-IEEE-LABEL: s_fdiv_v2f16:
; GFX9-IEEE: ; %bb.0:
@@ -3790,49 +4281,55 @@ define amdgpu_ps i16 @s_rsq_f16(i16 inreg %a.arg) {
define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX6-IEEE-LABEL: s_rsq_v2f16:
; GFX6-IEEE: ; %bb.0:
-; GFX6-IEEE-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: s_lshr_b32 s0, s0, 16
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v1, v1
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], v1, v1, -1.0
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, -1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[0:1], -1.0, v0, -1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_div_scale_f32 v7, vcc, -1.0, v1, -1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[0:1], -1.0, v0, -1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v5, 1.0
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[0:1], v1, v1, -1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v7, v5
; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v9, v5, v5
; GFX6-IEEE-NEXT: v_fma_f32 v4, v8, v4, v4
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v7, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v11, -v3, v9, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v10, -v2, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v9, v11, v5, v9
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v10, v4, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v9, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v8, v6
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v9
-; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[0:1]
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, -1.0
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v3, v4
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v5, v7, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v7, v8, v7, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v9, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v8, v8, v4, v9
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v8, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[0:1], -1.0, v1, -1.0
; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v4, v8
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v2, v6, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v2, v6
+; GFX6-IEEE-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
+; GFX6-IEEE-NEXT: v_fma_f32 v2, v3, v7, v2
+; GFX6-IEEE-NEXT: v_fma_f32 v3, -v5, v2, v6
+; GFX6-IEEE-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v3, v7, v2
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_readfirstlane_b32 s1, v1
; GFX6-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-IEEE-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX6-IEEE-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX6-IEEE-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-IEEE-NEXT: s_or_b32 s0, s0, s1
; GFX6-IEEE-NEXT: ; return to shader part epilog
;
; GFX6-FLUSH-LABEL: s_rsq_v2f16:
; GFX6-FLUSH: ; %bb.0:
-; GFX6-FLUSH-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-FLUSH-NEXT: s_lshr_b32 s0, s0, 16
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, s0
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v1, v1
@@ -3841,8 +4338,9 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, -1.0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, s[0:1], -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
@@ -3851,40 +4349,110 @@ define amdgpu_ps i32 @s_rsq_v2f16(i32 inreg %a.arg) {
; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-FLUSH-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, -1.0
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v1, -1.0
+; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s2, v0
+; GFX6-FLUSH-NEXT: v_div_scale_f32 v0, s[0:1], -1.0, v1, -1.0
+; GFX6-FLUSH-NEXT: s_or_b64 s[0:1], s[0:1], s[0:1]
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
-; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
-; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
-; GFX6-FLUSH-NEXT: v_fma_f32 v6, -v2, v5, v4
-; GFX6-FLUSH-NEXT: v_fma_f32 v5, v6, v3, v5
-; GFX6-FLUSH-NEXT: v_fma_f32 v2, -v2, v5, v4
+; GFX6-FLUSH-NEXT: v_fma_f32 v4, -v2, v3, 1.0
+; GFX6-FLUSH-NEXT: v_fma_f32 v3, v4, v3, v3
+; GFX6-FLUSH-NEXT: v_mul_f32_e32 v4, v0, v3
+; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v4, v0
+; GFX6-FLUSH-NEXT: v_fma_f32 v4, v5, v3, v4
+; GFX6-FLUSH-NEXT: v_fma_f32 v0, -v2, v4, v0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-FLUSH-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
-; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-FLUSH-NEXT: s_cselect_b64 vcc, exec, 0
+; GFX6-FLUSH-NEXT: v_div_fmas_f32 v0, v0, v3, v4
+; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v0, v1, -1.0
+; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-FLUSH-NEXT: s_bfe_u32 s1, s2, 0x100000
; GFX6-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-FLUSH-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX6-FLUSH-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-FLUSH-NEXT: s_or_b32 s0, s1, s0
; GFX6-FLUSH-NEXT: ; return to shader part epilog
;
-; GFX8-LABEL: s_rsq_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_rsq_f16_e32 v0, s0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: v_rsq_f16_e32 v1, s0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x8000
-; GFX8-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; GFX8-NEXT: v_xor_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-IEEE-LABEL: s_rsq_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX8-IEEE-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-IEEE-NEXT: v_sqrt_f16_e32 v1, s0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v2, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v3, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v6
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, -1.0, v2
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, -1.0, v3
+; GFX8-IEEE-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, s0, v6
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s0, v3
+; GFX8-IEEE-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, s0, v7
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v3, v1, -1.0
+; GFX8-IEEE-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-IEEE-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-IEEE-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-IEEE-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-IEEE-NEXT: s_or_b32 s0, s0, s1
+; GFX8-IEEE-NEXT: ; return to shader part epilog
+;
+; GFX8-FLUSH-LABEL: s_rsq_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: v_sqrt_f16_e32 v0, s0
+; GFX8-FLUSH-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-FLUSH-NEXT: v_sqrt_f16_e32 v1, s0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v2, v4, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v3, v5
+; GFX8-FLUSH-NEXT: s_and_b32 s0, s0, 0xff800000
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s1, v2
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, s0, v6
+; GFX8-FLUSH-NEXT: s_and_b32 s0, s1, 0xff800000
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, s0, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v2, v0, -1.0
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v1, -1.0
+; GFX8-FLUSH-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-FLUSH-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-FLUSH-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-FLUSH-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-FLUSH-NEXT: s_or_b32 s0, s0, s1
+; GFX8-FLUSH-NEXT: ; return to shader part epilog
;
; GFX9-IEEE-LABEL: s_rsq_v2f16:
; GFX9-IEEE: ; %bb.0:
@@ -4831,8 +5399,8 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX6-IEEE-LABEL: v_rsq_v2f16:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v1, v1
; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
@@ -4840,40 +5408,42 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, 1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, 1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, 1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_div_scale_f32 v7, vcc, 1.0, v1, 1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[4:5], 1.0, v0, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v5, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, 1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, 1.0, v1, 1.0
; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v9, v5, v5
; GFX6-IEEE-NEXT: v_fma_f32 v4, v8, v4, v4
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v7, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v11, -v3, v9, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v10, -v2, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v9, v11, v5, v9
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v10, v4, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v9, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v8, v6
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v9
-; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v3, v4
+; GFX6-IEEE-NEXT: v_div_scale_f32 v7, s[4:5], 1.0, v0, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v6, v8, v6, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v9, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v10, v7, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v8, v8, v4, v9
+; GFX6-IEEE-NEXT: v_fma_f32 v9, -v5, v10, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v8, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v9, v6, v10
; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v4, v8
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, 1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v3, v7
+; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[4:5]
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v6, v3
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_rsq_v2f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v1, v1
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
@@ -4900,7 +5470,6 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, 1.0, v0, 1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -4911,16 +5480,77 @@ define <2 x half> @v_rsq_v2f16(<2 x half> %a) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, 1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_rsq_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rsq_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rsq_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_rsq_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v6, -v2, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v7, -v3, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, 1.0, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, 1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v7
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, 1.0, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, 1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_rsq_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, -v2, v4, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, -v3, v5, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, 1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, 1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, 1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, 1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_rsq_v2f16:
; GFX9-IEEE: ; %bb.0:
@@ -5074,8 +5704,8 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX6-IEEE-LABEL: v_neg_rsq_v2f16:
; GFX6-IEEE: ; %bb.0:
; GFX6-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-IEEE-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v1, v1
; GFX6-IEEE-NEXT: v_sqrt_f32_e32 v0, v0
@@ -5083,40 +5713,42 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-IEEE-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v3, s[4:5], v1, v1, -1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v0, v0, -1.0
-; GFX6-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-IEEE-NEXT: v_div_scale_f32 v2, s[4:5], v1, v1, -1.0
; GFX6-IEEE-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-IEEE-NEXT: v_div_scale_f32 v7, vcc, -1.0, v1, -1.0
-; GFX6-IEEE-NEXT: v_div_scale_f32 v6, s[4:5], -1.0, v0, -1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v9, -v3, v5, 1.0
+; GFX6-IEEE-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, -1.0
+; GFX6-IEEE-NEXT: v_rcp_f32_e32 v6, v5
+; GFX6-IEEE-NEXT: v_div_scale_f32 v3, vcc, -1.0, v1, -1.0
; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v4, 1.0
-; GFX6-IEEE-NEXT: v_fma_f32 v5, v9, v5, v5
; GFX6-IEEE-NEXT: v_fma_f32 v4, v8, v4, v4
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v7, v5
-; GFX6-IEEE-NEXT: v_mul_f32_e32 v8, v6, v4
-; GFX6-IEEE-NEXT: v_fma_f32 v11, -v3, v9, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v10, -v2, v8, v6
-; GFX6-IEEE-NEXT: v_fma_f32 v9, v11, v5, v9
-; GFX6-IEEE-NEXT: v_fma_f32 v8, v10, v4, v8
-; GFX6-IEEE-NEXT: v_fma_f32 v3, -v3, v9, v7
-; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v8, v6
-; GFX6-IEEE-NEXT: v_div_fmas_f32 v3, v3, v5, v9
-; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[4:5]
-; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v3, v1, -1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v5, v6, 1.0
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v9, v3, v4
+; GFX6-IEEE-NEXT: v_div_scale_f32 v7, s[4:5], -1.0, v0, -1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v6, v8, v6, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v8, -v2, v9, v3
+; GFX6-IEEE-NEXT: v_mul_f32_e32 v10, v7, v6
+; GFX6-IEEE-NEXT: v_fma_f32 v8, v8, v4, v9
+; GFX6-IEEE-NEXT: v_fma_f32 v9, -v5, v10, v7
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v2, v8, v3
+; GFX6-IEEE-NEXT: v_fma_f32 v3, v9, v6, v10
; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v4, v8
-; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_div_fixup_f32 v1, v2, v1, -1.0
+; GFX6-IEEE-NEXT: v_fma_f32 v2, -v5, v3, v7
+; GFX6-IEEE-NEXT: s_mov_b64 vcc, s[4:5]
+; GFX6-IEEE-NEXT: v_div_fmas_f32 v2, v2, v6, v3
; GFX6-IEEE-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-IEEE-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-IEEE-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-IEEE-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-IEEE-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-IEEE-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-IEEE-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-FLUSH-LABEL: v_neg_rsq_v2f16:
; GFX6-FLUSH: ; %bb.0:
; GFX6-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-FLUSH-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-FLUSH-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v1, v1
; GFX6-FLUSH-NEXT: v_sqrt_f32_e32 v0, v0
@@ -5143,7 +5775,6 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX6-FLUSH-NEXT: v_rcp_f32_e32 v3, v2
; GFX6-FLUSH-NEXT: v_div_scale_f32 v4, vcc, -1.0, v0, -1.0
; GFX6-FLUSH-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-FLUSH-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FLUSH-NEXT: v_fma_f32 v3, v5, v3, v3
; GFX6-FLUSH-NEXT: v_mul_f32_e32 v5, v4, v3
@@ -5154,19 +5785,77 @@ define <2 x half> @v_neg_rsq_v2f16(<2 x half> %a) {
; GFX6-FLUSH-NEXT: v_div_fmas_f32 v2, v2, v3, v5
; GFX6-FLUSH-NEXT: v_div_fixup_f32 v0, v2, v0, -1.0
; GFX6-FLUSH-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-FLUSH-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-FLUSH-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-FLUSH-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-FLUSH-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_neg_rsq_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rsq_f16_e32 v1, v0
-; GFX8-NEXT: v_rsq_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x8000
-; GFX8-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX8-NEXT: v_xor_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-IEEE-LABEL: v_neg_rsq_v2f16:
+; GFX8-IEEE: ; %bb.0:
+; GFX8-IEEE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-IEEE-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX8-IEEE-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-IEEE-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-IEEE-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v2, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v3, v5
+; GFX8-IEEE-NEXT: v_add_f32_e32 v6, -1.0, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v7, -1.0, v7
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v6, v6, v4
+; GFX8-IEEE-NEXT: v_sub_f32_e32 v7, v7, v5
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v2, -v2, v6
+; GFX8-IEEE-NEXT: v_mul_f32_e64 v3, -v3, v7
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, -1.0, v2
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, -1.0, v3
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-IEEE-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-IEEE-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-IEEE-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-IEEE-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX8-IEEE-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-IEEE-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX8-IEEE-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX8-IEEE-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-IEEE-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-IEEE-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-IEEE-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-FLUSH-LABEL: v_neg_rsq_v2f16:
+; GFX8-FLUSH: ; %bb.0:
+; GFX8-FLUSH-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-FLUSH-NEXT: v_sqrt_f16_e32 v1, v0
+; GFX8-FLUSH-NEXT: v_sqrt_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX8-FLUSH-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v4, v2
+; GFX8-FLUSH-NEXT: v_rcp_f32_e32 v5, v3
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v2, v4, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v3, v5, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v6, v6, v4, -v4
+; GFX8-FLUSH-NEXT: v_mad_f32 v7, v7, v5, -v5
+; GFX8-FLUSH-NEXT: v_mad_f32 v2, -v2, v6, -1.0
+; GFX8-FLUSH-NEXT: v_mad_f32 v3, -v3, v7, -1.0
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v2, v2, v4
+; GFX8-FLUSH-NEXT: v_mul_f32_e32 v3, v3, v5
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v2, 0xff800000, v2
+; GFX8-FLUSH-NEXT: v_and_b32_e32 v3, 0xff800000, v3
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v2, v2, v6
+; GFX8-FLUSH-NEXT: v_add_f32_e32 v3, v3, v7
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX8-FLUSH-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v1, v2, v1, -1.0
+; GFX8-FLUSH-NEXT: v_div_fixup_f16 v0, v3, v0, -1.0
+; GFX8-FLUSH-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-FLUSH-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-FLUSH-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-FLUSH-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-IEEE-LABEL: v_neg_rsq_v2f16:
; GFX9-IEEE: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
index 78eb3c28eb42d..7505234fae8d2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fma.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu9.0a < %s | FileCheck -check-prefix=GFX90A %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10 %s
@@ -364,116 +364,33 @@ define <2 x half> @v_fma_v2f16(<2 x half> %x, <2 x half> %y, <2 x half> %z) {
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v3
-; GFX6-NEXT: s_movk_i32 s4, 0x3f1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[3:4], v5
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v6
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[7:8], v7
-; GFX6-NEXT: s_movk_i32 s5, 0xfc10
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v1
-; GFX6-NEXT: v_fma_f64 v[3:4], v[7:8], v[5:6], v[3:4]
-; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0x1ff, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffe, v5
-; GFX6-NEXT: v_bfe_u32 v6, v4, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_sub_i32_e32 v7, vcc, s4, v6
-; GFX6-NEXT: v_or_b32_e32 v5, 0x1000, v3
-; GFX6-NEXT: v_med3_i32 v7, v7, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, v7, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v7, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, s5, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 12, v6
-; GFX6-NEXT: v_or_b32_e32 v5, v8, v5
-; GFX6-NEXT: v_or_b32_e32 v7, v3, v7
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_and_b32_e32 v7, 7, v5
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 2, v5
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7c00
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_mov_b32_e32 v8, 0x7e00
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: s_movk_i32 s6, 0x40f
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v5, v3, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v9
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0x8000, v4
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v11
-; GFX6-NEXT: v_fma_f64 v[0:1], v[5:6], v[2:3], v[0:1]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0x1ff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v4, v1, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v0
-; GFX6-NEXT: v_med3_i32 v5, v5, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, v5, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: v_or_b32_e32 v5, v0, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_fma_f32 v1, v3, v4, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fma_v2f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_fma_f16 v3, v5, v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX8-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_fma_f16 v1, v3, v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fma_v2f16:
@@ -518,119 +435,37 @@ define <2 x half> @v_fma_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y, <2 x half>
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v5
-; GFX6-NEXT: s_movk_i32 s4, 0x3f1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[3:4], v4
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v6
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[7:8], v7
-; GFX6-NEXT: s_movk_i32 s5, 0xfc10
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_fma_f64 v[3:4], v[7:8], v[5:6], v[3:4]
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v1
-; GFX6-NEXT: v_and_b32_e32 v5, 0x1ff, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffe, v5
-; GFX6-NEXT: v_bfe_u32 v6, v4, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_sub_i32_e32 v7, vcc, s4, v6
-; GFX6-NEXT: v_or_b32_e32 v5, 0x1000, v3
-; GFX6-NEXT: v_med3_i32 v7, v7, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, v7, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v7, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, s5, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 12, v6
-; GFX6-NEXT: v_or_b32_e32 v5, v8, v5
-; GFX6-NEXT: v_or_b32_e32 v7, v3, v7
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_and_b32_e32 v7, 7, v5
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 2, v5
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7c00
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_mov_b32_e32 v8, 0x7e00
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: s_movk_i32 s6, 0x40f
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v5, v3, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0x8000, v4
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v10
-; GFX6-NEXT: v_fma_f64 v[0:1], v[0:1], v[5:6], v[2:3]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0x1ff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v4, v1, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v0
-; GFX6-NEXT: v_med3_i32 v5, v5, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, v5, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: v_or_b32_e32 v5, v0, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_fma_f32 v1, v3, v4, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fma_v2f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_fma_f16 v3, -v5, v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_fma_f16 v0, -v0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX8-NEXT: v_fma_f16 v1, v3, v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fma_v2f16_fneg_lhs:
@@ -677,118 +512,36 @@ define <2 x half> @v_fma_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y, <2 x half>
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v5
-; GFX6-NEXT: s_movk_i32 s4, 0x3f1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[3:4], v4
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v6
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[7:8], v7
-; GFX6-NEXT: s_movk_i32 s5, 0xfc10
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_fma_f64 v[3:4], v[5:6], v[7:8], v[3:4]
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0x1ff, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffe, v5
-; GFX6-NEXT: v_bfe_u32 v6, v4, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_sub_i32_e32 v7, vcc, s4, v6
-; GFX6-NEXT: v_or_b32_e32 v5, 0x1000, v3
-; GFX6-NEXT: v_med3_i32 v7, v7, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, v7, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v7, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, s5, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 12, v6
-; GFX6-NEXT: v_or_b32_e32 v5, v8, v5
-; GFX6-NEXT: v_or_b32_e32 v7, v3, v7
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_and_b32_e32 v7, 7, v5
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 2, v5
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7c00
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_mov_b32_e32 v8, 0x7e00
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: s_movk_i32 s6, 0x40f
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v5, v3, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0x8000, v4
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v10
-; GFX6-NEXT: v_fma_f64 v[0:1], v[5:6], v[0:1], v[2:3]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0x1ff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v4, v1, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v0
-; GFX6-NEXT: v_med3_i32 v5, v5, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, v5, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: v_or_b32_e32 v5, v0, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_fma_f32 v1, v3, v4, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fma_v2f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_fma_f16 v3, v5, -v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_fma_f16 v0, v0, -v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX8-NEXT: v_fma_f16 v1, v3, v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fma_v2f16_fneg_rhs:
@@ -836,116 +589,33 @@ define <2 x half> @v_fma_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y, <2 x h
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v3
-; GFX6-NEXT: s_movk_i32 s4, 0x3f1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[3:4], v5
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v6
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[7:8], v7
-; GFX6-NEXT: s_movk_i32 s5, 0xfc10
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v1
-; GFX6-NEXT: v_fma_f64 v[3:4], v[7:8], v[5:6], v[3:4]
-; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0x1ff, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v4
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffe, v5
-; GFX6-NEXT: v_bfe_u32 v6, v4, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_sub_i32_e32 v7, vcc, s4, v6
-; GFX6-NEXT: v_or_b32_e32 v5, 0x1000, v3
-; GFX6-NEXT: v_med3_i32 v7, v7, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, v7, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v7, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, s5, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 12, v6
-; GFX6-NEXT: v_or_b32_e32 v5, v8, v5
-; GFX6-NEXT: v_or_b32_e32 v7, v3, v7
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_and_b32_e32 v7, 7, v5
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 2, v5
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v7
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7c00
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v5, v7, v5, vcc
-; GFX6-NEXT: v_mov_b32_e32 v8, 0x7e00
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v3
-; GFX6-NEXT: s_movk_i32 s6, 0x40f
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v6
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v5, v3, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v9
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 0x8000, v4
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v11
-; GFX6-NEXT: v_fma_f64 v[0:1], v[5:6], v[2:3], v[0:1]
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v3, 0x1ff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v4, v1, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v0
-; GFX6-NEXT: v_med3_i32 v5, v5, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, v5, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v6, v3
-; GFX6-NEXT: v_or_b32_e32 v5, v0, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v7, v8, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_fma_f32 v1, v3, v4, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fma_v2f16_fneg_lhs_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_fma_f16 v3, v5, v4, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX8-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_fma_f16 v1, v3, v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fma_v2f16_fneg_lhs_rhs:
@@ -991,166 +661,44 @@ define <3 x half> @v_fma_v3f16(<3 x half> %x, <3 x half> %y, <3 x half> %z) {
; GFX6-LABEL: v_fma_v3f16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v8, v8
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v7
-; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v6
-; GFX6-NEXT: s_movk_i32 s4, 0x3f1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[6:7], v8
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[8:9], v9
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[10:11], v10
-; GFX6-NEXT: s_movk_i32 s5, 0xfc10
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-NEXT: v_fma_f64 v[6:7], v[10:11], v[8:9], v[6:7]
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_and_b32_e32 v8, 0x1ff, v7
-; GFX6-NEXT: v_or_b32_e32 v6, v8, v6
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 8, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffe, v8
-; GFX6-NEXT: v_bfe_u32 v9, v7, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v6, v8, v6
-; GFX6-NEXT: v_sub_i32_e32 v10, vcc, s4, v9
-; GFX6-NEXT: v_or_b32_e32 v8, 0x1000, v6
-; GFX6-NEXT: v_med3_i32 v10, v10, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, v10, v8
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, v10, v11
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v10, v8
-; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v9, vcc, s5, v9
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 12, v9
-; GFX6-NEXT: v_or_b32_e32 v8, v11, v8
-; GFX6-NEXT: v_or_b32_e32 v10, v6, v10
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v9
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX6-NEXT: v_and_b32_e32 v10, 7, v8
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v10
-; GFX6-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v10
-; GFX6-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v10, v10, v11
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 2, v8
-; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v10
-; GFX6-NEXT: v_mov_b32_e32 v12, 0x7c00
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v9
-; GFX6-NEXT: v_cndmask_b32_e32 v8, v12, v8, vcc
-; GFX6-NEXT: v_mov_b32_e32 v13, 0x7e00
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; GFX6-NEXT: s_movk_i32 s6, 0x40f
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v12, v13, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v9
-; GFX6-NEXT: v_cndmask_b32_e32 v14, v8, v6, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[5:6], v5
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[8:9], v3
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[10:11], v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v14
-; GFX6-NEXT: v_fma_f64 v[5:6], v[10:11], v[8:9], v[5:6]
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0x1ff, v6
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v5
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v6
-; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v5, v6, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX6-NEXT: v_sub_i32_e32 v8, vcc, s4, v5
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v1
-; GFX6-NEXT: v_med3_i32 v8, v8, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, v8, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, v8, v9
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v8, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, s5, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, 12, v5
-; GFX6-NEXT: v_or_b32_e32 v3, v9, v3
-; GFX6-NEXT: v_or_b32_e32 v8, v1, v8
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v5
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v8, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v8, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v8
-; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v8
-; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v8, v8, v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v8
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v8, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v8, v0
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v5
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v12, v3, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v12, v13, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v5
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v3, v1, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v4
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[4:5], v8
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX6-NEXT: v_and_b32_e32 v6, 0x8000, v6
-; GFX6-NEXT: v_or_b32_e32 v6, v6, v9
-; GFX6-NEXT: v_fma_f64 v[2:3], v[4:5], v[2:3], v[0:1]
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v6
-; GFX6-NEXT: v_and_b32_e32 v0, 0x1ff, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 8, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffe, v2
-; GFX6-NEXT: v_bfe_u32 v4, v3, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
-; GFX6-NEXT: v_or_b32_e32 v2, 0x1000, v0
-; GFX6-NEXT: v_med3_i32 v5, v5, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, v5, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v5, v2
-; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v4
-; GFX6-NEXT: v_or_b32_e32 v2, v6, v2
-; GFX6-NEXT: v_or_b32_e32 v5, v0, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v2
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 2, v2
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v2, v12, v2, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v12, v13, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v2, 0x8000, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX6-NEXT: v_fma_f32 v6, v6, v7, v8
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
+; GFX6-NEXT: v_fma_f32 v1, v1, v3, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v2, v6, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v7
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fma_v3f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GFX8-NEXT: v_fma_f16 v6, v8, v7, v6
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GFX8-NEXT: v_fma_f16 v0, v0, v2, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX8-NEXT: v_fma_f16 v2, v6, v7, v8
; GFX8-NEXT: v_fma_f16 v1, v1, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fma_v3f16:
@@ -1199,222 +747,60 @@ define <4 x half> @v_fma_v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z) {
; GFX6-LABEL: v_fma_v4f16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v6
; GFX6-NEXT: v_cvt_f32_f16_e32 v8, v8
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v7
-; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v6
-; GFX6-NEXT: s_movk_i32 s4, 0x3f1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[6:7], v8
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[8:9], v9
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[10:11], v10
-; GFX6-NEXT: s_movk_i32 s5, 0xfc10
-; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; GFX6-NEXT: v_fma_f64 v[7:8], v[10:11], v[8:9], v[6:7]
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v6, 0x1ff, v8
-; GFX6-NEXT: v_or_b32_e32 v6, v6, v7
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 8, v8
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffe, v7
-; GFX6-NEXT: v_or_b32_e32 v10, v7, v6
-; GFX6-NEXT: v_bfe_u32 v7, v8, 20, 11
-; GFX6-NEXT: v_sub_i32_e32 v11, vcc, s4, v7
-; GFX6-NEXT: v_or_b32_e32 v6, 0x1000, v10
-; GFX6-NEXT: v_med3_i32 v11, v11, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v14, v11, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v11, v11, v14
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v11, v6
-; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v11, vcc, s5, v7
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 12, v11
-; GFX6-NEXT: v_or_b32_e32 v6, v14, v6
-; GFX6-NEXT: v_or_b32_e32 v7, v10, v7
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v11
-; GFX6-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
-; GFX6-NEXT: v_and_b32_e32 v7, 7, v6
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v7
-; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v7, v7, v14
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 2, v6
-; GFX6-NEXT: v_add_i32_e32 v7, vcc, v6, v7
-; GFX6-NEXT: v_cvt_f32_f16_e32 v9, v9
-; GFX6-NEXT: v_cvt_f32_f16_e32 v13, v13
-; GFX6-NEXT: v_cvt_f32_f16_e32 v15, v12
-; GFX6-NEXT: v_mov_b32_e32 v6, 0x7c00
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v11
-; GFX6-NEXT: v_cndmask_b32_e32 v14, v6, v7, vcc
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7e00
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; GFX6-NEXT: s_movk_i32 s6, 0x40f
-; GFX6-NEXT: v_cndmask_b32_e32 v10, v6, v7, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v11
-; GFX6-NEXT: v_cndmask_b32_e32 v16, v14, v10, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[9:10], v9
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[11:12], v13
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[13:14], v15
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v10, v10
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; GFX6-NEXT: v_fma_f32 v0, v0, v2, v4
+; GFX6-NEXT: v_fma_f32 v2, v6, v8, v10
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_fma_f64 v[9:10], v[13:14], v[11:12], v[9:10]
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX6-NEXT: v_and_b32_e32 v11, 0x1ff, v10
-; GFX6-NEXT: v_or_b32_e32 v9, v11, v9
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 8, v10
-; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v11, 0xffe, v11
-; GFX6-NEXT: v_bfe_u32 v12, v10, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v9, v11, v9
-; GFX6-NEXT: v_sub_i32_e32 v13, vcc, s4, v12
-; GFX6-NEXT: v_or_b32_e32 v11, 0x1000, v9
-; GFX6-NEXT: v_med3_i32 v13, v13, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v14, v13, v11
-; GFX6-NEXT: v_lshlrev_b32_e32 v13, v13, v14
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v13, v11
-; GFX6-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v12, vcc, s5, v12
-; GFX6-NEXT: v_lshlrev_b32_e32 v13, 12, v12
-; GFX6-NEXT: v_or_b32_e32 v11, v14, v11
-; GFX6-NEXT: v_or_b32_e32 v13, v9, v13
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v12
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v13, v11, vcc
-; GFX6-NEXT: v_and_b32_e32 v13, 7, v11
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v13
-; GFX6-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v13
-; GFX6-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v13, v13, v14
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 2, v11
-; GFX6-NEXT: v_add_i32_e32 v11, vcc, v11, v13
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v12
-; GFX6-NEXT: v_cndmask_b32_e32 v11, v6, v11, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v9
-; GFX6-NEXT: v_and_b32_e32 v8, 0x8000, v8
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v6, v7, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v12
-; GFX6-NEXT: v_or_b32_e32 v8, v8, v16
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[11:12], v5
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[13:14], v3
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[15:16], v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v10
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX6-NEXT: v_fma_f64 v[9:10], v[15:16], v[13:14], v[11:12]
-; GFX6-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0x1ff, v10
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v9
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v10
-; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v5, v10, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX6-NEXT: v_sub_i32_e32 v9, vcc, s4, v5
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v1
-; GFX6-NEXT: v_med3_i32 v9, v9, 0, 13
-; GFX6-NEXT: v_lshrrev_b32_e32 v12, v9, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v9, v9, v12
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v9, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v9, vcc, s5, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v9
-; GFX6-NEXT: v_or_b32_e32 v3, v12, v3
-; GFX6-NEXT: v_or_b32_e32 v5, v1, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v9
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v12
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v9
-; GFX6-NEXT: v_cndmask_b32_e32 v12, v6, v3, vcc
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v0
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v3
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; GFX6-NEXT: v_cndmask_b32_e32 v13, v6, v7, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v9
-; GFX6-NEXT: v_cndmask_b32_e32 v9, v12, v13, vcc
-; GFX6-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
-; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v10
-; GFX6-NEXT: v_and_b32_e32 v3, 0x1ff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffe, v3
-; GFX6-NEXT: v_bfe_u32 v4, v1, 20, 11
-; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v4
-; GFX6-NEXT: v_and_b32_e32 v10, 0x8000, v10
-; GFX6-NEXT: v_or_b32_e32 v3, 0x1000, v0
-; GFX6-NEXT: v_med3_i32 v5, v5, 0, 13
-; GFX6-NEXT: v_or_b32_e32 v2, v10, v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, v5, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v5, v9
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v5, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 12, v4
-; GFX6-NEXT: v_or_b32_e32 v3, v9, v3
-; GFX6-NEXT: v_or_b32_e32 v5, v0, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 1, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX6-NEXT: v_and_b32_e32 v5, 7, v3
-; GFX6-NEXT: v_cmp_lt_i32_e32 vcc, 5, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, 3, v5
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v9
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 2, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
-; GFX6-NEXT: v_cmp_gt_i32_e32 vcc, 31, v4
-; GFX6-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
-; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v6, v7, vcc
-; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s6, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX6-NEXT: v_and_b32_e32 v1, 0x8000, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v8
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v5
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v7
+; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v9
+; GFX6-NEXT: v_cvt_f32_f16_e32 v7, v11
+; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_fma_f32 v1, v1, v3, v4
+; GFX6-NEXT: v_fma_f32 v3, v5, v6, v7
+; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fma_v4f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX8-NEXT: v_fma_f16 v6, v8, v7, v6
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX8-NEXT: v_fma_f16 v7, v9, v8, v7
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v5
; GFX8-NEXT: v_fma_f16 v0, v0, v2, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v7
+; GFX8-NEXT: v_fma_f16 v2, v6, v8, v10
+; GFX8-NEXT: v_mov_b32_e32 v4, 16
; GFX8-NEXT: v_fma_f16 v1, v1, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_fma_f16 v3, v7, v9, v11
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_fma_v4f16:
@@ -2283,120 +1669,43 @@ define amdgpu_ps <2 x half> @fma_v2s16_uniform(<2 x half> inreg %a, <2 x half> i
; GFX6-NEXT: s_lshr_b32 s3, s0, 16
; GFX6-NEXT: s_lshr_b32 s4, s1, 16
; GFX6-NEXT: s_lshr_b32 s5, s2, 16
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, s4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, s3
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; GFX6-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, s1
-; GFX6-NEXT: v_readfirstlane_b32 s3, v1
-; GFX6-NEXT: s_and_b32 s4, s3, 0x1ff
-; GFX6-NEXT: v_readfirstlane_b32 s5, v0
-; GFX6-NEXT: s_or_b32 s4, s4, s5
-; GFX6-NEXT: s_cselect_b32 s4, 1, 0
-; GFX6-NEXT: s_lshr_b32 s5, s3, 8
-; GFX6-NEXT: s_bfe_u32 s6, s3, 0xb0014
-; GFX6-NEXT: s_and_b32 s5, s5, 0xffe
-; GFX6-NEXT: s_sub_i32 s7, 0x3f1, s6
-; GFX6-NEXT: s_or_b32 s4, s5, s4
-; GFX6-NEXT: v_med3_i32 v0, s7, 0, 13
-; GFX6-NEXT: s_or_b32 s5, s4, 0x1000
-; GFX6-NEXT: v_readfirstlane_b32 s7, v0
-; GFX6-NEXT: s_lshr_b32 s8, s5, s7
-; GFX6-NEXT: s_lshl_b32 s7, s8, s7
-; GFX6-NEXT: s_cmp_lg_u32 s7, s5
-; GFX6-NEXT: s_cselect_b32 s5, 1, 0
-; GFX6-NEXT: s_addk_i32 s6, 0xfc10
-; GFX6-NEXT: s_lshl_b32 s7, s6, 12
-; GFX6-NEXT: s_or_b32 s5, s8, s5
-; GFX6-NEXT: s_or_b32 s7, s4, s7
-; GFX6-NEXT: s_cmp_lt_i32 s6, 1
-; GFX6-NEXT: s_cselect_b32 s5, s5, s7
-; GFX6-NEXT: s_and_b32 s7, s5, 7
-; GFX6-NEXT: s_cmp_gt_i32 s7, 5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, s0
-; GFX6-NEXT: s_cselect_b32 s8, 1, 0
-; GFX6-NEXT: s_cmp_eq_u32 s7, 3
-; GFX6-NEXT: s_cselect_b32 s7, 1, 0
-; GFX6-NEXT: s_or_b32 s7, s7, s8
-; GFX6-NEXT: s_lshr_b32 s5, s5, 2
-; GFX6-NEXT: s_add_i32 s5, s5, s7
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[2:3], v2
-; GFX6-NEXT: v_cvt_f64_f32_e32 v[4:5], v4
-; GFX6-NEXT: s_cmp_lt_i32 s6, 31
-; GFX6-NEXT: s_cselect_b32 s5, s5, 0x7c00
-; GFX6-NEXT: s_cmp_lg_u32 s4, 0
-; GFX6-NEXT: s_movk_i32 s4, 0x7e00
-; GFX6-NEXT: s_cselect_b32 s7, s4, 0x7c00
-; GFX6-NEXT: s_cmpk_eq_i32 s6, 0x40f
-; GFX6-NEXT: s_cselect_b32 s0, s7, s5
-; GFX6-NEXT: s_lshr_b32 s1, s3, 16
-; GFX6-NEXT: v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
-; GFX6-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX6-NEXT: s_or_b32 s0, s1, s0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, s1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, s2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, s3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v5, s5
+; GFX6-NEXT: v_fma_f32 v0, v0, v1, v2
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_fma_f32 v1, v3, v4, v5
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_bfe_u32 s0, s0, 0x100000
; GFX6-NEXT: v_readfirstlane_b32 s1, v1
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_and_b32 s2, s1, 0x1ff
-; GFX6-NEXT: v_readfirstlane_b32 s3, v0
-; GFX6-NEXT: s_or_b32 s2, s2, s3
-; GFX6-NEXT: s_cselect_b32 s2, 1, 0
-; GFX6-NEXT: s_lshr_b32 s3, s1, 8
-; GFX6-NEXT: s_bfe_u32 s5, s1, 0xb0014
-; GFX6-NEXT: s_and_b32 s3, s3, 0xffe
-; GFX6-NEXT: s_sub_i32 s6, 0x3f1, s5
-; GFX6-NEXT: s_or_b32 s2, s3, s2
-; GFX6-NEXT: v_med3_i32 v0, s6, 0, 13
-; GFX6-NEXT: s_or_b32 s3, s2, 0x1000
-; GFX6-NEXT: v_readfirstlane_b32 s6, v0
-; GFX6-NEXT: s_lshr_b32 s7, s3, s6
-; GFX6-NEXT: s_lshl_b32 s6, s7, s6
-; GFX6-NEXT: s_cmp_lg_u32 s6, s3
-; GFX6-NEXT: s_cselect_b32 s3, 1, 0
-; GFX6-NEXT: s_addk_i32 s5, 0xfc10
-; GFX6-NEXT: s_lshl_b32 s6, s5, 12
-; GFX6-NEXT: s_or_b32 s3, s7, s3
-; GFX6-NEXT: s_or_b32 s6, s2, s6
-; GFX6-NEXT: s_cmp_lt_i32 s5, 1
-; GFX6-NEXT: s_cselect_b32 s3, s3, s6
-; GFX6-NEXT: s_and_b32 s6, s3, 7
-; GFX6-NEXT: s_cmp_gt_i32 s6, 5
-; GFX6-NEXT: s_cselect_b32 s7, 1, 0
-; GFX6-NEXT: s_cmp_eq_u32 s6, 3
-; GFX6-NEXT: s_cselect_b32 s6, 1, 0
-; GFX6-NEXT: s_or_b32 s6, s6, s7
-; GFX6-NEXT: s_lshr_b32 s3, s3, 2
-; GFX6-NEXT: s_add_i32 s3, s3, s6
-; GFX6-NEXT: s_cmp_lt_i32 s5, 31
-; GFX6-NEXT: s_cselect_b32 s3, s3, 0x7c00
-; GFX6-NEXT: s_cmp_lg_u32 s2, 0
-; GFX6-NEXT: s_cselect_b32 s2, s4, 0x7c00
-; GFX6-NEXT: s_cmpk_eq_i32 s5, 0x40f
-; GFX6-NEXT: s_cselect_b32 s2, s2, s3
-; GFX6-NEXT: s_lshr_b32 s1, s1, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0x8000
-; GFX6-NEXT: s_or_b32 s1, s1, s2
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s1, s0
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: fma_v2s16_uniform:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s3, s2, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s2
; GFX8-NEXT: s_lshr_b32 s4, s1, 16
-; GFX8-NEXT: s_lshr_b32 s5, s0, 16
+; GFX8-NEXT: s_lshr_b32 s5, s2, 16
+; GFX8-NEXT: v_fma_f16 v0, s0, v0, v1
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_fma_f16 v0, s5, v0, v1
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_fma_f16 v1, s0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_fma_f16 v0, s3, v0, v1
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: fma_v2s16_uniform:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
index 69b642e41e7f4..34d4e6d60d35a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-mesa3d < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-mesa3d < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.70-amd-mesa3d -mattr=-real-true16 < %s | FileCheck -check-prefix=GFX1170 %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-mesa3d -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-TRUE16 %s
@@ -549,13 +549,14 @@ define <2 x half> @test_min_max_v2f16(<2 x half> %a) #0 {
; GFX8-LABEL: test_min_max_v2f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x4000
-; GFX8-NEXT: v_max_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e32 v0, 2.0, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, 0x4400
+; GFX8-NEXT: v_mov_b32_e32 v2, 0x4000
+; GFX8-NEXT: v_max_f16_e32 v1, 2.0, v0
+; GFX8-NEXT: v_max_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_min_f16_e32 v0, 4.0, v0
-; GFX8-NEXT: v_min_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_min_f16_e32 v1, 4.0, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_min_max_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
index ce734c8531264..fd5ebda9dbc34 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmul.v2f16.ll
@@ -14,9 +14,11 @@ define <2 x half> @v_fmul_v2f16(<2 x half> %a, <2 x half> %b) {
; GFX8-LABEL: v_fmul_v2f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v2f16:
@@ -38,9 +40,12 @@ define <2 x half> @v_fmul_v2f16_fneg_lhs(<2 x half> %a, <2 x half> %b) {
; GFX8-LABEL: v_fmul_v2f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, -v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v0, -v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v2f16_fneg_lhs:
@@ -63,9 +68,12 @@ define <2 x half> @v_fmul_v2f16_fneg_rhs(<2 x half> %a, <2 x half> %b) {
; GFX8-LABEL: v_fmul_v2f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, -v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v2f16_fneg_rhs:
@@ -88,9 +96,11 @@ define <2 x half> @v_fmul_v2f16_fneg_lhs_fneg_rhs(<2 x half> %a, <2 x half> %b)
; GFX8-LABEL: v_fmul_v2f16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v2f16_fneg_lhs_fneg_rhs:
@@ -116,11 +126,14 @@ define <2 x half> @v_fmul_v2f16_partial_neg(<2 x half> %a, <2 x half> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT: v_mul_f16_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v1, v0
-; GFX8-NEXT: v_mul_f16_sdwa v2, -v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_mul_f16_e64 v0, -v1, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v1
+; GFX8-NEXT: v_mul_f16_e32 v3, v1, v0
+; GFX8-NEXT: v_mul_f16_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v1, v2, v3
+; GFX8-NEXT: v_mul_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v2f16_partial_neg:
@@ -211,10 +224,13 @@ define <3 x half> @v_fmul_v3f16(<3 x half> %a, <3 x half> %b) {
; GFX8-LABEL: v_fmul_v3f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v3f16:
@@ -238,10 +254,15 @@ define <3 x half> @v_fmul_v3f16_fneg_lhs(<3 x half> %a, <3 x half> %b) {
; GFX8-LABEL: v_fmul_v3f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, -v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v0, -v0, v2
-; GFX8-NEXT: v_mul_f16_e64 v1, -v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v3f16_fneg_lhs:
@@ -266,10 +287,15 @@ define <3 x half> @v_fmul_v3f16_fneg_rhs(<3 x half> %a, <3 x half> %b) {
; GFX8-LABEL: v_fmul_v3f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v0, -v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v2
-; GFX8-NEXT: v_mul_f16_e64 v1, v1, -v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, 0x80008000, v3
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v3f16_fneg_rhs:
@@ -294,10 +320,13 @@ define <3 x half> @v_fmul_v3f16_fneg_lhs_fneg_rhs(<3 x half> %a, <3 x half> %b)
; GFX8-LABEL: v_fmul_v3f16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v3f16_fneg_lhs_fneg_rhs:
@@ -323,12 +352,15 @@ define <4 x half> @v_fmul_v4f16(<4 x half> %a, <4 x half> %b) {
; GFX8-LABEL: v_fmul_v4f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v1, v3
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v4f16:
@@ -352,12 +384,17 @@ define <4 x half> @v_fmul_v4f16_fneg_lhs(<4 x half> %a, <4 x half> %b) {
; GFX8-LABEL: v_fmul_v4f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, -v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v5, -v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v1, -v1, v3
-; GFX8-NEXT: v_mul_f16_e64 v0, -v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v1, v3
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v4f16_fneg_lhs:
@@ -382,12 +419,17 @@ define <4 x half> @v_fmul_v4f16_fneg_rhs(<4 x half> %a, <4 x half> %b) {
; GFX8-LABEL: v_fmul_v4f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v1, -v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v5, v0, -v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v1, v1, -v3
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, 0x80008000, v3
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v1, v3
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v4f16_fneg_rhs:
@@ -412,12 +454,15 @@ define <4 x half> @v_fmul_v4f16_fneg_lhs_fneg_rhs(<4 x half> %a, <4 x half> %b)
; GFX8-LABEL: v_fmul_v4f16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v2, v1, v3
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v4f16_fneg_lhs_fneg_rhs:
@@ -444,15 +489,19 @@ define <6 x half> @v_fmul_v6f16(<6 x half> %a, <6 x half> %b) {
; GFX8-LABEL: v_fmul_v6f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v6, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v7, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v8, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v2, v2, v5
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v4
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT: v_mul_f16_e32 v6, v0, v3
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v3, v1, v4
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v2, v5
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v5, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v6f16:
@@ -478,15 +527,22 @@ define <6 x half> @v_fmul_v6f16_fneg_lhs(<6 x half> %a, <6 x half> %b) {
; GFX8-LABEL: v_fmul_v6f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v6, -v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v7, -v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v8, -v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v2, -v2, v5
-; GFX8-NEXT: v_mul_f16_e64 v1, -v1, v4
-; GFX8-NEXT: v_mul_f16_e64 v0, -v0, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT: v_mul_f16_e32 v6, v0, v3
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v3, v1, v4
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v2, v5
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v5, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v6f16_fneg_lhs:
@@ -513,15 +569,22 @@ define <6 x half> @v_fmul_v6f16_fneg_rhs(<6 x half> %a, <6 x half> %b) {
; GFX8-LABEL: v_fmul_v6f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v6, v2, -v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v7, v1, -v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v8, v0, -v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v2, v2, -v5
-; GFX8-NEXT: v_mul_f16_e64 v1, v1, -v4
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT: v_xor_b32_e32 v3, 0x80008000, v3
+; GFX8-NEXT: v_xor_b32_e32 v4, 0x80008000, v4
+; GFX8-NEXT: v_xor_b32_e32 v5, 0x80008000, v5
+; GFX8-NEXT: v_mul_f16_e32 v6, v0, v3
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v3, v1, v4
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v2, v5
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v5, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v6f16_fneg_rhs:
@@ -548,15 +611,19 @@ define <6 x half> @v_fmul_v6f16_fneg_lhs_fneg_rhs(<6 x half> %a, <6 x half> %b)
; GFX8-LABEL: v_fmul_v6f16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v6, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v7, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v8, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v2, v2, v5
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v4
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-NEXT: v_mul_f16_e32 v6, v0, v3
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v3, v1, v4
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v2, v5
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v5, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v6f16_fneg_lhs_fneg_rhs:
@@ -585,18 +652,23 @@ define <8 x half> @v_fmul_v8f16(<8 x half> %a, <8 x half> %b) {
; GFX8-LABEL: v_fmul_v8f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v8, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v9, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v10, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v11, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_mul_f16_e32 v2, v2, v6
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v5
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v10
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX8-NEXT: v_mul_f16_e32 v8, v0, v4
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v1, v5
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v5, v2, v6
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v6, v3, v7
+; GFX8-NEXT: v_mul_f16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v7, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v8f16:
@@ -624,18 +696,27 @@ define <8 x half> @v_fmul_v8f16_fneg_lhs(<8 x half> %a, <8 x half> %b) {
; GFX8-LABEL: v_fmul_v8f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v8, -v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v9, -v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v10, -v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v11, -v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v3, -v3, v7
-; GFX8-NEXT: v_mul_f16_e64 v2, -v2, v6
-; GFX8-NEXT: v_mul_f16_e64 v1, -v1, v5
-; GFX8-NEXT: v_mul_f16_e64 v0, -v0, v4
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v10
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT: v_xor_b32_e32 v3, 0x80008000, v3
+; GFX8-NEXT: v_mul_f16_e32 v8, v0, v4
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v1, v5
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v5, v2, v6
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v6, v3, v7
+; GFX8-NEXT: v_mul_f16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v7, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v8f16_fneg_lhs:
@@ -664,18 +745,27 @@ define <8 x half> @v_fmul_v8f16_fneg_rhs(<8 x half> %a, <8 x half> %b) {
; GFX8-LABEL: v_fmul_v8f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v8, v3, -v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v9, v2, -v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v10, v1, -v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v11, v0, -v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e64 v3, v3, -v7
-; GFX8-NEXT: v_mul_f16_e64 v2, v2, -v6
-; GFX8-NEXT: v_mul_f16_e64 v1, v1, -v5
-; GFX8-NEXT: v_mul_f16_e64 v0, v0, -v4
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v10
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX8-NEXT: v_xor_b32_e32 v4, 0x80008000, v4
+; GFX8-NEXT: v_xor_b32_e32 v5, 0x80008000, v5
+; GFX8-NEXT: v_xor_b32_e32 v6, 0x80008000, v6
+; GFX8-NEXT: v_xor_b32_e32 v7, 0x80008000, v7
+; GFX8-NEXT: v_mul_f16_e32 v8, v0, v4
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v1, v5
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v5, v2, v6
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v6, v3, v7
+; GFX8-NEXT: v_mul_f16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v7, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v8f16_fneg_rhs:
@@ -704,18 +794,23 @@ define <8 x half> @v_fmul_v8f16_fneg_lhs_fneg_rhs(<8 x half> %a, <8 x half> %b)
; GFX8-LABEL: v_fmul_v8f16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v8, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v9, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v10, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v11, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v3, v3, v7
-; GFX8-NEXT: v_mul_f16_e32 v2, v2, v6
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v5
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v10
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v9
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v8
+; GFX8-NEXT: v_mul_f16_e32 v8, v0, v4
+; GFX8-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v4, v1, v5
+; GFX8-NEXT: v_mul_f16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v5, v2, v6
+; GFX8-NEXT: v_mul_f16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mul_f16_e32 v6, v3, v7
+; GFX8-NEXT: v_mul_f16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v7, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_fmul_v8f16_fneg_lhs_fneg_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
index 17fff47248f44..fc7c4a57fe32e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fpow.ll
@@ -200,40 +200,57 @@ define <2 x half> @v_pow_v2f16(<2 x half> %x, <2 x half> %y) {
; GFX6-LABEL: v_pow_v2f16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX6-NEXT: v_log_f32_e32 v2, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_log_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_mov_b32_e32 v3, 0xc2fc0000
+; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v2, v4
+; GFX6-NEXT: v_mov_b32_e32 v5, 0x42800000
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v2, v3
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v5, vcc
+; GFX6-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX6-NEXT: v_not_b32_e32 v4, 63
+; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v4, vcc
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc
; GFX6-NEXT: v_exp_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v1
; GFX6-NEXT: v_exp_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX6-NEXT: v_ldexp_f32_e32 v1, v2, v6
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_pow_v2f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_log_f16_e32 v2, v0
+; GFX8-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX8-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v2
; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX8-NEXT: v_log_f32_e32 v2, v2
-; GFX8-NEXT: v_log_f32_e32 v0, v0
-; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX8-NEXT: v_exp_f32_e32 v1, v2
-; GFX8-NEXT: v_exp_f32_e32 v0, v0
-; GFX8-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_exp_f16_e32 v0, v0
+; GFX8-NEXT: v_exp_f16_e32 v1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_pow_v2f16:
@@ -327,40 +344,58 @@ define <2 x half> @v_pow_v2f16_fneg_lhs(<2 x half> %x, <2 x half> %y) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v1
; GFX6-NEXT: v_log_f32_e32 v2, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_log_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_mov_b32_e32 v3, 0xc2fc0000
+; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v2, v4
+; GFX6-NEXT: v_mov_b32_e32 v5, 0x42800000
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v2, v3
+; GFX6-NEXT: v_cndmask_b32_e32 v4, 0, v5, vcc
+; GFX6-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX6-NEXT: v_not_b32_e32 v4, 63
+; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v4, vcc
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v0, v3
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc
; GFX6-NEXT: v_exp_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v1
; GFX6-NEXT: v_exp_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX6-NEXT: v_ldexp_f32_e32 v1, v2, v6
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v4, vcc
+; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_pow_v2f16_fneg_lhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX8-NEXT: v_log_f32_e32 v2, v2
-; GFX8-NEXT: v_log_f32_e32 v0, v0
-; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX8-NEXT: v_exp_f32_e32 v1, v2
-; GFX8-NEXT: v_exp_f32_e32 v0, v0
-; GFX8-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_log_f16_e32 v2, v0
+; GFX8-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX8-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_exp_f16_e32 v0, v0
+; GFX8-NEXT: v_exp_f16_e32 v1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_pow_v2f16_fneg_lhs:
@@ -460,41 +495,59 @@ define <2 x half> @v_pow_v2f16_fneg_rhs(<2 x half> %x, <2 x half> %y) {
; GFX6-LABEL: v_pow_v2f16_fneg_rhs:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v1
; GFX6-NEXT: v_log_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_log_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_mov_b32_e32 v4, 0xc2fc0000
+; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX6-NEXT: v_mov_b32_e32 v3, 0x42800000
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v2, v4
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX6-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX6-NEXT: v_not_b32_e32 v5, 63
+; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v5, vcc
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
; GFX6-NEXT: v_exp_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v1
; GFX6-NEXT: v_exp_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX6-NEXT: v_ldexp_f32_e32 v1, v2, v6
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc
+; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_pow_v2f16_fneg_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_log_f16_e32 v2, v0
+; GFX8-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX8-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX8-NEXT: v_log_f32_e32 v2, v2
-; GFX8-NEXT: v_log_f32_e32 v0, v0
-; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX8-NEXT: v_exp_f32_e32 v1, v2
-; GFX8-NEXT: v_exp_f32_e32 v0, v0
-; GFX8-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_exp_f16_e32 v0, v0
+; GFX8-NEXT: v_exp_f16_e32 v1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_pow_v2f16_fneg_rhs:
@@ -594,41 +647,60 @@ define <2 x half> @v_pow_v2f16_fneg_lhs_rhs(<2 x half> %x, <2 x half> %y) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v1
; GFX6-NEXT: v_log_f32_e32 v2, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
; GFX6-NEXT: v_log_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
+; GFX6-NEXT: v_mov_b32_e32 v4, 0xc2fc0000
+; GFX6-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
+; GFX6-NEXT: v_mov_b32_e32 v3, 0x42800000
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v2, v4
+; GFX6-NEXT: v_cndmask_b32_e32 v5, 0, v3, vcc
+; GFX6-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX6-NEXT: v_not_b32_e32 v5, 63
+; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX6-NEXT: v_cndmask_b32_e32 v6, 0, v5, vcc
+; GFX6-NEXT: v_cmp_lt_f32_e32 vcc, v0, v4
+; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v3, vcc
; GFX6-NEXT: v_exp_f32_e32 v2, v2
-; GFX6-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
+; GFX6-NEXT: v_add_f32_e32 v0, v0, v1
; GFX6-NEXT: v_exp_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX6-NEXT: v_ldexp_f32_e32 v1, v2, v6
+; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc
+; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_pow_v2f16_fneg_lhs_rhs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v2, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; GFX8-NEXT: v_cvt_f32_f16_sdwa v3, -v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_cvt_f32_f16_e64 v1, -v1
-; GFX8-NEXT: v_log_f32_e32 v2, v2
-; GFX8-NEXT: v_log_f32_e32 v0, v0
-; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v3, v2
-; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v1, v0
-; GFX8-NEXT: v_exp_f32_e32 v1, v2
-; GFX8-NEXT: v_exp_f32_e32 v0, v0
-; GFX8-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_log_f16_e32 v2, v0
+; GFX8-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v3, v1
+; GFX8-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX8-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX8-NEXT: v_mul_legacy_f32_e32 v0, v0, v1
+; GFX8-NEXT: v_mul_legacy_f32_e32 v2, v2, v3
; GFX8-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_exp_f16_e32 v0, v0
+; GFX8-NEXT: v_exp_f16_e32 v1, v1
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_pow_v2f16_fneg_lhs_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
index 0fea02b5fe904..0bfad55132a82 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshl.ll
@@ -999,91 +999,161 @@ define i16 @v_fshl_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
define amdgpu_ps i32 @s_fshl_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 inreg %amt.arg) {
; GFX6-LABEL: s_fshl_v4i8:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_and_b32 s9, s2, 7
+; GFX6-NEXT: s_lshr_b32 s3, s0, 8
; GFX6-NEXT: s_lshr_b32 s4, s0, 16
-; GFX6-NEXT: s_and_b32 s5, s0, 0xff00
-; GFX6-NEXT: s_bfe_u32 s6, s1, 0x80008
-; GFX6-NEXT: s_and_b32 s7, s1, 0xff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 8
-; GFX6-NEXT: s_or_b32 s0, s0, s7
-; GFX6-NEXT: s_and_b32 s7, s2, 7
-; GFX6-NEXT: s_or_b32 s5, s5, s6
-; GFX6-NEXT: s_bfe_u32 s6, s2, 0x30008
-; GFX6-NEXT: s_lshl_b32 s0, s0, s7
-; GFX6-NEXT: s_lshl_b32 s5, s5, s6
-; GFX6-NEXT: s_bfe_u32 s0, s0, 0x80008
-; GFX6-NEXT: s_and_b32 s5, s5, 0xff00
-; GFX6-NEXT: s_lshr_b32 s3, s1, 24
-; GFX6-NEXT: s_or_b32 s0, s0, s5
-; GFX6-NEXT: s_and_b32 s5, s4, 0xff00
-; GFX6-NEXT: s_lshl_b32 s4, s4, 8
-; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX6-NEXT: s_or_b32 s3, s5, s3
-; GFX6-NEXT: s_bfe_u32 s5, s2, 0x30018
-; GFX6-NEXT: s_or_b32 s1, s4, s1
-; GFX6-NEXT: s_bfe_u32 s2, s2, 0x30010
-; GFX6-NEXT: s_lshl_b32 s3, s3, s5
-; GFX6-NEXT: s_lshl_b32 s1, s1, s2
-; GFX6-NEXT: s_and_b32 s3, s3, 0xff00
-; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80008
-; GFX6-NEXT: s_or_b32 s1, s1, s3
+; GFX6-NEXT: s_lshr_b32 s5, s0, 24
+; GFX6-NEXT: s_lshr_b32 s6, s2, 8
+; GFX6-NEXT: s_lshr_b32 s7, s2, 16
+; GFX6-NEXT: s_lshr_b32 s8, s2, 24
+; GFX6-NEXT: s_andn2_b32 s2, 7, s2
+; GFX6-NEXT: s_lshl_b32 s0, s0, s9
+; GFX6-NEXT: s_bfe_u32 s9, s1, 0x70001
+; GFX6-NEXT: s_lshr_b32 s2, s9, s2
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s6, 7
+; GFX6-NEXT: s_lshl_b32 s2, s3, s2
+; GFX6-NEXT: s_bfe_u32 s3, s1, 0x80008
+; GFX6-NEXT: s_andn2_b32 s6, 7, s6
+; GFX6-NEXT: s_lshr_b32 s3, s3, 1
+; GFX6-NEXT: s_lshr_b32 s3, s3, s6
+; GFX6-NEXT: s_or_b32 s2, s2, s3
+; GFX6-NEXT: s_and_b32 s3, s7, 7
+; GFX6-NEXT: s_lshl_b32 s3, s4, s3
+; GFX6-NEXT: s_bfe_u32 s4, s1, 0x80010
+; GFX6-NEXT: s_andn2_b32 s6, 7, s7
+; GFX6-NEXT: s_lshr_b32 s4, s4, 1
+; GFX6-NEXT: s_lshr_b32 s4, s4, s6
+; GFX6-NEXT: s_or_b32 s3, s3, s4
+; GFX6-NEXT: s_and_b32 s4, s8, 7
+; GFX6-NEXT: s_andn2_b32 s6, 7, s8
+; GFX6-NEXT: s_lshr_b32 s1, s1, 25
+; GFX6-NEXT: s_lshl_b32 s4, s5, s4
+; GFX6-NEXT: s_lshr_b32 s1, s1, s6
+; GFX6-NEXT: s_or_b32 s4, s4, s1
+; GFX6-NEXT: s_and_b32 s1, s2, 0xff
+; GFX6-NEXT: s_and_b32 s0, s0, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s3, 0xff
; GFX6-NEXT: s_lshl_b32 s1, s1, 16
; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s4, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_v4i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0xc0c0105
-; GFX8-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xc0c0004
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX8-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX8-NEXT: s_and_b32 s0, s2, 7
-; GFX8-NEXT: s_mov_b32 s3, 0xc0c0105
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, s0, v2
-; GFX8-NEXT: v_perm_b32 v1, v2, v1, s3
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xc0c0307
-; GFX8-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX8-NEXT: v_mov_b32_e32 v3, 0xc0c0206
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, s0, v2
-; GFX8-NEXT: v_perm_b32 v0, s1, v0, v3
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30010
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_perm_b32 v0, v0, v2, s3
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_and_b32 s11, s2, 7
+; GFX8-NEXT: s_lshr_b32 s3, s0, 8
+; GFX8-NEXT: s_lshr_b32 s4, s0, 16
+; GFX8-NEXT: s_lshr_b32 s5, s0, 24
+; GFX8-NEXT: s_lshl_b32 s0, s0, s11
+; GFX8-NEXT: s_and_b32 s11, s1, 0xff
+; GFX8-NEXT: s_lshr_b32 s8, s2, 8
+; GFX8-NEXT: s_lshr_b32 s9, s2, 16
+; GFX8-NEXT: s_lshr_b32 s10, s2, 24
+; GFX8-NEXT: s_and_b32 s11, 0xffff, s11
+; GFX8-NEXT: s_xor_b32 s2, s2, -1
+; GFX8-NEXT: s_lshr_b32 s11, s11, 1
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_lshr_b32 s2, s11, s2
+; GFX8-NEXT: s_lshr_b32 s6, s1, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, s8, 7
+; GFX8-NEXT: s_lshl_b32 s2, s3, s2
+; GFX8-NEXT: s_and_b32 s3, s6, 0xff
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_xor_b32 s6, s8, -1
+; GFX8-NEXT: s_lshr_b32 s3, s3, 1
+; GFX8-NEXT: s_and_b32 s6, s6, 7
+; GFX8-NEXT: s_lshr_b32 s3, s3, s6
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_and_b32 s3, s9, 7
+; GFX8-NEXT: s_lshl_b32 s3, s4, s3
+; GFX8-NEXT: s_and_b32 s4, s7, 0xff
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_xor_b32 s6, s9, -1
+; GFX8-NEXT: s_lshr_b32 s4, s4, 1
+; GFX8-NEXT: s_and_b32 s6, s6, 7
+; GFX8-NEXT: s_lshr_b32 s4, s4, s6
+; GFX8-NEXT: s_or_b32 s3, s3, s4
+; GFX8-NEXT: s_and_b32 s4, s10, 7
+; GFX8-NEXT: s_lshl_b32 s4, s5, s4
+; GFX8-NEXT: s_xor_b32 s5, s10, -1
+; GFX8-NEXT: s_lshr_b32 s1, s1, 25
+; GFX8-NEXT: s_and_b32 s5, s5, 7
+; GFX8-NEXT: s_and_b32 s2, s2, 0xff
+; GFX8-NEXT: s_lshr_b32 s1, s1, s5
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s2, s2, 8
+; GFX8-NEXT: s_or_b32 s1, s4, s1
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, s3, 0xff
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_lshl_b32 s1, s1, 24
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v4i8:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0004
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v3, s1, s0, v3
-; GFX11-NEXT: v_mov_b32_e32 v2, 0xc0c0206
-; GFX11-NEXT: v_perm_b32 v2, s1, s0, v2
-; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0307
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX11-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX11-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX11-NEXT: s_bfe_u32 s1, s2, 0x30010
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, s1, v2
-; GFX11-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX11-NEXT: s_and_b32 s1, s2, 7
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, s1, v3
-; GFX11-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_perm_b32 v0, v3, v0, 0xc0c0105
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_and_b32 s11, s1, 0xff
+; GFX11-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-NEXT: s_lshr_b32 s10, s2, 24
+; GFX11-NEXT: s_and_b32 s12, s2, 7
+; GFX11-NEXT: s_and_b32 s11, 0xffff, s11
+; GFX11-NEXT: s_xor_b32 s2, s2, -1
+; GFX11-NEXT: s_lshr_b32 s11, s11, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: s_lshr_b32 s2, s11, s2
+; GFX11-NEXT: s_and_b32 s11, s8, 7
+; GFX11-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-NEXT: s_xor_b32 s8, s8, -1
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_lshr_b32 s6, s6, 1
+; GFX11-NEXT: s_and_b32 s8, s8, 7
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s0, 24
+; GFX11-NEXT: s_lshr_b32 s7, s1, 16
+; GFX11-NEXT: s_lshl_b32 s0, s0, s12
+; GFX11-NEXT: s_lshl_b32 s3, s3, s11
+; GFX11-NEXT: s_lshr_b32 s6, s6, s8
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s2, s3, s6
+; GFX11-NEXT: s_and_b32 s3, s7, 0xff
+; GFX11-NEXT: s_xor_b32 s7, s9, -1
+; GFX11-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX11-NEXT: s_and_b32 s7, s7, 7
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_and_b32 s6, s9, 7
+; GFX11-NEXT: s_lshr_b32 s3, s3, s7
+; GFX11-NEXT: s_xor_b32 s7, s10, -1
+; GFX11-NEXT: s_lshl_b32 s4, s4, s6
+; GFX11-NEXT: s_and_b32 s6, s10, 7
+; GFX11-NEXT: s_lshr_b32 s1, s1, 25
+; GFX11-NEXT: s_and_b32 s7, s7, 7
+; GFX11-NEXT: s_lshl_b32 s5, s5, s6
+; GFX11-NEXT: s_lshr_b32 s1, s1, s7
+; GFX11-NEXT: s_or_b32 s3, s4, s3
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: s_or_b32 s1, s5, s1
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s2, s3, 16
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v4i8:
; GFX9: ; %bb.0:
@@ -1199,34 +1269,47 @@ define i32 @v_fshl_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX6-LABEL: v_fshl_v4i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_and_b32_e32 v9, 7, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xff00, v0
-; GFX6-NEXT: v_bfe_u32 v6, v1, 8, 8
-; GFX6-NEXT: v_and_b32_e32 v7, 0xff, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v7
-; GFX6-NEXT: v_and_b32_e32 v7, 7, v2
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX6-NEXT: v_bfe_u32 v6, v2, 8, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, v7, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, v6, v5
-; GFX6-NEXT: v_bfe_u32 v0, v0, 8, 8
-; GFX6-NEXT: v_and_b32_e32 v5, 0xff00, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 24, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX6-NEXT: v_and_b32_e32 v5, 0xff00, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v4
-; GFX6-NEXT: v_bfe_u32 v1, v1, 16, 8
-; GFX6-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_bfe_u32 v5, v2, 24, 3
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 8, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 24, v2
+; GFX6-NEXT: v_bfi_b32 v2, v2, 0, 7
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v9, v0
+; GFX6-NEXT: v_bfe_u32 v9, v1, 1, 7
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v9
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 7, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_bfe_u32 v3, v1, 8, 8
+; GFX6-NEXT: v_bfi_b32 v6, v6, 0, 7
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 1, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v6, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 7, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_bfe_u32 v4, v1, 16, 8
+; GFX6-NEXT: v_bfi_b32 v6, v7, 0, 7
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 1, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v6, v4
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 7, v8
+; GFX6-NEXT: v_bfi_b32 v6, v8, 0, 7
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 25, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v6, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v2
; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX6-NEXT: v_bfe_u32 v2, v2, 16, 3
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, v5, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, v2, v1
-; GFX6-NEXT: v_and_b32_e32 v3, 0xff00, v3
-; GFX6-NEXT: v_bfe_u32 v1, v1, 8, 8
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
@@ -1753,290 +1836,365 @@ define i24 @v_fshl_i24(i24 %lhs, i24 %rhs, i24 %amt) {
define amdgpu_ps i48 @s_fshl_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 inreg %amt.arg) {
; GFX6-LABEL: s_fshl_v2i24:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_getpc_b64 s[8:9]
-; GFX6-NEXT: s_mov_b32 s8, s0
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX6-NEXT: v_mov_b32_e32 v0, s5
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_add_u32 s8, s8, s6
-; GFX6-NEXT: s_addc_u32 s9, s9, 0
-; GFX6-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:20
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:4
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshr_b32 s7, s1, 8
+; GFX6-NEXT: s_bfe_u32 s9, s0, 0x80008
+; GFX6-NEXT: s_and_b32 s1, s1, 0xff
; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s2
-; GFX6-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:8
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:20
-; GFX6-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:21
-; GFX6-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:4
-; GFX6-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:5
-; GFX6-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:13
-; GFX6-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:12
-; GFX6-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:18
-; GFX6-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:2
-; GFX6-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:3
-; GFX6-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:10
-; GFX6-NEXT: buffer_load_ushort v10, off, s[8:11], 0 offset:8
-; GFX6-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:19
-; GFX6-NEXT: buffer_load_ushort v12, off, s[8:11], 0 offset:16
-; GFX6-NEXT: buffer_load_ushort v13, off, s[8:11], 0
-; GFX6-NEXT: buffer_load_ubyte v14, off, s[8:11], 0 offset:11
-; GFX6-NEXT: s_mov_b32 s0, 0xaaaaaab
-; GFX6-NEXT: s_waitcnt vmcnt(14)
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX6-NEXT: s_waitcnt vmcnt(13)
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: s_waitcnt vmcnt(12)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX6-NEXT: s_waitcnt vmcnt(11)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: s_waitcnt vmcnt(10)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v4
-; GFX6-NEXT: s_waitcnt vmcnt(9)
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX6-NEXT: s_waitcnt vmcnt(8)
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX6-NEXT: s_waitcnt vmcnt(7)
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX6-NEXT: s_waitcnt vmcnt(6)
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v8
-; GFX6-NEXT: s_waitcnt vmcnt(5)
-; GFX6-NEXT: v_lshlrev_b32_e32 v9, 24, v9
-; GFX6-NEXT: s_waitcnt vmcnt(4)
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX6-NEXT: s_waitcnt vmcnt(3)
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX6-NEXT: s_waitcnt vmcnt(2)
-; GFX6-NEXT: v_or_b32_e32 v6, v12, v6
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v6, v6, s0
-; GFX6-NEXT: v_mul_hi_u32 v0, v0, s0
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v14
-; GFX6-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v5
-; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 3, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX6-NEXT: v_or_b32_e32 v7, v13, v7
-; GFX6-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, v12, v4
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, v11, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v13, v9, 1
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 1, v7
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX6-NEXT: v_alignbit_b32 v2, v8, v2, 1
-; GFX6-NEXT: v_not_b32_e32 v4, v4
-; GFX6-NEXT: v_not_b32_e32 v0, v0
-; GFX6-NEXT: v_alignbit_b32 v3, v5, v3, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v1, v2, v0
-; GFX6-NEXT: buffer_store_short v3, off, s[8:11], 0 offset:24
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:27
-; GFX6-NEXT: buffer_store_byte v1, off, s[8:11], 0 offset:26
-; GFX6-NEXT: buffer_store_byte v2, off, s[8:11], 0 offset:29
-; GFX6-NEXT: buffer_store_byte v3, off, s[8:11], 0 offset:28
-; GFX6-NEXT: s_waitcnt expcnt(3)
-; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX6-NEXT: s_waitcnt expcnt(2)
-; GFX6-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: s_and_b32 s8, s0, 0xff
+; GFX6-NEXT: s_lshl_b32 s9, s9, 8
+; GFX6-NEXT: s_and_b32 s6, s6, 0xff
+; GFX6-NEXT: v_alignbit_b32 v0, s1, v0, 24
+; GFX6-NEXT: s_and_b32 s1, s7, 0xff
+; GFX6-NEXT: s_or_b32 s8, s8, s9
+; GFX6-NEXT: s_and_b32 s6, 0xffff, s6
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_readfirstlane_b32 s1, v1
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_lshr_b32 s7, s3, 8
+; GFX6-NEXT: s_and_b32 s3, s3, 0xff
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX6-NEXT: s_lshl_b32 s6, s6, 16
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: v_alignbit_b32 v0, s3, v0, 24
+; GFX6-NEXT: s_or_b32 s6, s8, s6
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_lshr_b32 s1, s2, 16
+; GFX6-NEXT: s_and_b32 s8, s2, 0xff
+; GFX6-NEXT: s_bfe_u32 s9, s2, 0x80008
+; GFX6-NEXT: v_readfirstlane_b32 s2, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s3, s7, 0xff
+; GFX6-NEXT: s_lshl_b32 s9, s9, 8
+; GFX6-NEXT: s_and_b32 s1, s1, 0xff
+; GFX6-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT: s_or_b32 s8, s8, s9
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s3
+; GFX6-NEXT: s_lshr_b32 s3, s4, 16
+; GFX6-NEXT: s_bfe_u32 s9, s4, 0x80008
+; GFX6-NEXT: s_or_b32 s1, s8, s1
+; GFX6-NEXT: s_and_b32 s8, s4, 0xff
+; GFX6-NEXT: s_lshl_b32 s9, s9, 8
+; GFX6-NEXT: s_and_b32 s3, s3, 0xff
+; GFX6-NEXT: s_or_b32 s8, s8, s9
+; GFX6-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX6-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, s4
+; GFX6-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-NEXT: s_or_b32 s3, s8, s3
+; GFX6-NEXT: s_mul_i32 s8, s4, 0xffffffe8
+; GFX6-NEXT: v_mul_hi_u32 v0, v0, s8
+; GFX6-NEXT: s_lshr_b32 s7, s5, 8
+; GFX6-NEXT: s_and_b32 s5, s5, 0xff
+; GFX6-NEXT: s_and_b32 s7, s7, 0xff
+; GFX6-NEXT: v_readfirstlane_b32 s8, v0
+; GFX6-NEXT: s_add_i32 s4, s4, s8
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT: v_alignbit_b32 v1, s5, v1, 24
+; GFX6-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX6-NEXT: v_readfirstlane_b32 s5, v1
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_or_b32 s5, s5, s7
+; GFX6-NEXT: v_readfirstlane_b32 s7, v0
+; GFX6-NEXT: s_mulk_i32 s7, 0xffe8
+; GFX6-NEXT: s_add_i32 s3, s3, s7
+; GFX6-NEXT: s_cmp_ge_u32 s3, 24
+; GFX6-NEXT: s_cselect_b32 s7, 1, 0
+; GFX6-NEXT: s_sub_i32 s8, s3, 24
+; GFX6-NEXT: s_cmp_lg_u32 s7, 0
+; GFX6-NEXT: s_cselect_b32 s3, s8, s3
+; GFX6-NEXT: s_cmp_ge_u32 s3, 24
+; GFX6-NEXT: s_cselect_b32 s7, 1, 0
+; GFX6-NEXT: s_sub_i32 s8, s3, 24
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: s_cmp_lg_u32 s7, 0
+; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0
+; GFX6-NEXT: s_cselect_b32 s3, s8, s3
+; GFX6-NEXT: s_sub_i32 s7, 23, s3
+; GFX6-NEXT: s_lshr_b32 s1, s1, 1
+; GFX6-NEXT: s_lshl_b32 s3, s6, s3
+; GFX6-NEXT: s_lshr_b32 s1, s1, s7
+; GFX6-NEXT: s_or_b32 s3, s3, s1
+; GFX6-NEXT: v_readfirstlane_b32 s1, v0
+; GFX6-NEXT: s_mulk_i32 s1, 0xffe8
+; GFX6-NEXT: s_add_i32 s5, s5, s1
+; GFX6-NEXT: s_cmp_ge_u32 s5, 24
+; GFX6-NEXT: s_cselect_b32 s1, 1, 0
+; GFX6-NEXT: s_sub_i32 s4, s5, 24
+; GFX6-NEXT: s_cmp_lg_u32 s1, 0
+; GFX6-NEXT: s_cselect_b32 s1, s4, s5
+; GFX6-NEXT: s_cmp_ge_u32 s1, 24
+; GFX6-NEXT: s_cselect_b32 s4, 1, 0
+; GFX6-NEXT: s_sub_i32 s5, s1, 24
+; GFX6-NEXT: s_cmp_lg_u32 s4, 0
+; GFX6-NEXT: s_cselect_b32 s1, s5, s1
+; GFX6-NEXT: s_sub_i32 s4, 23, s1
+; GFX6-NEXT: s_lshl_b32 s0, s0, s1
+; GFX6-NEXT: s_lshr_b32 s1, s2, 1
+; GFX6-NEXT: s_lshr_b32 s1, s1, s4
+; GFX6-NEXT: s_bfe_u32 s2, s3, 0x80008
+; GFX6-NEXT: s_or_b32 s1, s0, s1
+; GFX6-NEXT: s_and_b32 s0, s3, 0xff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 8
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_bfe_u32 s2, s3, 0x80010
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s1, 0xff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_bfe_u32 s2, s1, 0x80008
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX6-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-NEXT: s_or_b32 s1, s2, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_v2i24:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_getpc_b64 s[8:9]
-; GFX8-NEXT: s_mov_b32 s8, s0
-; GFX8-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_u32 s8, s8, s6
-; GFX8-NEXT: s_addc_u32 s9, s9, 0
-; GFX8-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:20
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:4
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:8
-; GFX8-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:18
-; GFX8-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:2
-; GFX8-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:20
-; GFX8-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:21
-; GFX8-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:5
-; GFX8-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:13
-; GFX8-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:12
-; GFX8-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:4
-; GFX8-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:3
-; GFX8-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:11
-; GFX8-NEXT: buffer_load_ubyte v10, off, s[8:11], 0 offset:10
-; GFX8-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:19
-; GFX8-NEXT: s_and_b32 s3, s4, 0xffff
-; GFX8-NEXT: s_mov_b32 s1, 0xaaaaaab
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_and_b32 s5, s0, 0xffff
-; GFX8-NEXT: s_lshl_b32 s2, s2, 8
-; GFX8-NEXT: s_waitcnt vmcnt(11)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_or_b32_e32 v0, s3, v0
-; GFX8-NEXT: s_waitcnt vmcnt(9)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: s_waitcnt vmcnt(8)
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s1
-; GFX8-NEXT: s_waitcnt vmcnt(7)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: s_waitcnt vmcnt(5)
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX8-NEXT: s_waitcnt vmcnt(4)
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 8, v7
-; GFX8-NEXT: s_waitcnt vmcnt(3)
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v9
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v11
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX8-NEXT: v_mul_hi_u32 v2, v2, s1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v10, 24, v10
-; GFX8-NEXT: v_or_b32_e32 v3, v7, v4
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 8, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 3, v2
-; GFX8-NEXT: v_or_b32_e32 v1, s5, v1
-; GFX8-NEXT: v_or_b32_e32 v9, s2, v10
-; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, v11, v2
-; GFX8-NEXT: v_alignbit_b32 v6, s0, v9, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 1, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 1, v3
-; GFX8-NEXT: v_alignbit_b32 v4, v8, v4, 1
-; GFX8-NEXT: v_not_b32_e32 v0, v0
-; GFX8-NEXT: v_not_b32_e32 v2, v2
-; GFX8-NEXT: v_alignbit_b32 v0, v1, v6, v0
-; GFX8-NEXT: v_alignbit_b32 v1, v3, v4, v2
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:24
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX8-NEXT: buffer_store_byte v1, off, s[8:11], 0 offset:27
-; GFX8-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:26
-; GFX8-NEXT: buffer_store_byte v2, off, s[8:11], 0 offset:29
-; GFX8-NEXT: buffer_store_byte v3, off, s[8:11], 0 offset:28
-; GFX8-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX8-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-NEXT: s_lshr_b32 s6, s0, 8
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_lshr_b32 s7, s0, 16
+; GFX8-NEXT: s_lshr_b32 s8, s0, 24
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s6, s6, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s6
+; GFX8-NEXT: s_and_b32 s6, s7, 0xff
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX8-NEXT: s_lshr_b32 s9, s1, 8
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s6, s6, 16
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_or_b32 s0, s0, s6
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_and_b32 s6, s9, 0xff
+; GFX8-NEXT: s_or_b32 s1, s8, s1
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s6, s6, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s6
+; GFX8-NEXT: s_lshr_b32 s6, s2, 8
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_lshr_b32 s7, s2, 16
+; GFX8-NEXT: s_lshr_b32 s8, s2, 24
+; GFX8-NEXT: s_and_b32 s2, s2, 0xff
+; GFX8-NEXT: s_lshl_b32 s6, s6, 8
+; GFX8-NEXT: s_or_b32 s2, s2, s6
+; GFX8-NEXT: s_and_b32 s6, s7, 0xff
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX8-NEXT: s_lshr_b32 s9, s3, 8
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s6, s6, 16
+; GFX8-NEXT: s_and_b32 s3, s3, 0xff
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT: s_or_b32 s2, s2, s6
+; GFX8-NEXT: s_lshl_b32 s3, s3, 8
+; GFX8-NEXT: s_and_b32 s6, s9, 0xff
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_or_b32 s3, s8, s3
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s6, s6, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s6
+; GFX8-NEXT: s_lshr_b32 s6, s4, 8
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: s_lshr_b32 s7, s4, 16
+; GFX8-NEXT: s_lshr_b32 s8, s4, 24
+; GFX8-NEXT: s_and_b32 s4, s4, 0xff
+; GFX8-NEXT: s_lshl_b32 s6, s6, 8
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: s_or_b32 s4, s4, s6
+; GFX8-NEXT: s_and_b32 s6, s7, 0xff
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_lshl_b32 s6, s6, 16
+; GFX8-NEXT: s_or_b32 s4, s4, s6
+; GFX8-NEXT: v_readfirstlane_b32 s6, v0
+; GFX8-NEXT: s_mul_i32 s7, s6, 0xffffffe8
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s7
+; GFX8-NEXT: s_lshr_b32 s9, s5, 8
+; GFX8-NEXT: s_and_b32 s5, s5, 0xff
+; GFX8-NEXT: s_lshl_b32 s5, s5, 8
+; GFX8-NEXT: s_or_b32 s5, s8, s5
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: s_add_i32 s6, s6, s8
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT: s_and_b32 s7, s9, 0xff
+; GFX8-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX8-NEXT: s_lshl_b32 s7, s7, 16
+; GFX8-NEXT: s_or_b32 s5, s5, s7
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
+; GFX8-NEXT: s_mulk_i32 s7, 0xffe8
+; GFX8-NEXT: s_add_i32 s4, s4, s7
+; GFX8-NEXT: s_cmp_ge_u32 s4, 24
+; GFX8-NEXT: s_cselect_b32 s7, 1, 0
+; GFX8-NEXT: s_sub_i32 s8, s4, 24
+; GFX8-NEXT: s_cmp_lg_u32 s7, 0
+; GFX8-NEXT: s_cselect_b32 s4, s8, s4
+; GFX8-NEXT: s_cmp_ge_u32 s4, 24
+; GFX8-NEXT: s_cselect_b32 s7, 1, 0
+; GFX8-NEXT: s_sub_i32 s8, s4, 24
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: s_cmp_lg_u32 s7, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s5, v0
+; GFX8-NEXT: s_cselect_b32 s4, s8, s4
+; GFX8-NEXT: s_sub_i32 s7, 23, s4
+; GFX8-NEXT: s_lshr_b32 s2, s2, 1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s4
+; GFX8-NEXT: s_lshr_b32 s2, s2, s7
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: s_mulk_i32 s2, 0xffe8
+; GFX8-NEXT: s_add_i32 s5, s5, s2
+; GFX8-NEXT: s_cmp_ge_u32 s5, 24
+; GFX8-NEXT: s_cselect_b32 s2, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s5, 24
+; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s5
+; GFX8-NEXT: s_cmp_ge_u32 s2, 24
+; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: s_sub_i32 s5, s2, 24
+; GFX8-NEXT: s_cmp_lg_u32 s4, 0
+; GFX8-NEXT: s_cselect_b32 s2, s5, s2
+; GFX8-NEXT: s_sub_i32 s4, 23, s2
+; GFX8-NEXT: s_lshl_b32 s1, s1, s2
+; GFX8-NEXT: s_lshr_b32 s2, s3, 1
+; GFX8-NEXT: s_lshr_b32 s2, s2, s4
+; GFX8-NEXT: s_bfe_u32 s3, s0, 0x80008
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s3, s3, 8
+; GFX8-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_or_b32 s0, s2, s0
+; GFX8-NEXT: s_and_b32 s2, s1, 0xff
+; GFX8-NEXT: s_lshl_b32 s2, s2, 24
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_bfe_u32 s2, s1, 0x80008
+; GFX8-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_or_b32 s1, s2, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v2i24:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
-; GFX11-NEXT: v_mov_b32_e32 v2, s1
-; GFX11-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX11-NEXT: scratch_store_b32 off, v0, off offset:16
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-NEXT: s_lshr_b32 s8, s0, 24
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_clause 0x4
-; GFX11-NEXT: scratch_store_b32 off, v0, off
-; GFX11-NEXT: scratch_store_b16 off, v1, off offset:20
-; GFX11-NEXT: scratch_store_b16 off, v2, off offset:4
-; GFX11-NEXT: scratch_store_b32 off, v3, off offset:8
-; GFX11-NEXT: scratch_store_b16 off, v4, off offset:12
-; GFX11-NEXT: s_clause 0xb
-; GFX11-NEXT: scratch_load_u8 v0, off, off offset:21
-; GFX11-NEXT: scratch_load_u8 v1, off, off offset:20
-; GFX11-NEXT: scratch_load_u8 v2, off, off offset:19
-; GFX11-NEXT: scratch_load_u8 v3, off, off offset:11
-; GFX11-NEXT: scratch_load_u8 v4, off, off offset:12
-; GFX11-NEXT: scratch_load_u8 v5, off, off offset:18
-; GFX11-NEXT: scratch_load_u8 v6, off, off offset:5
-; GFX11-NEXT: scratch_load_u8 v7, off, off offset:4
-; GFX11-NEXT: scratch_load_u8 v8, off, off offset:3
-; GFX11-NEXT: scratch_load_u8 v9, off, off offset:13
-; GFX11-NEXT: scratch_load_u8 v10, off, off offset:2
-; GFX11-NEXT: scratch_load_u8 v11, off, off offset:10
-; GFX11-NEXT: s_waitcnt vmcnt(11)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_waitcnt vmcnt(10)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-NEXT: s_waitcnt vmcnt(9)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v0, v1, v2, v0
-; GFX11-NEXT: s_waitcnt vmcnt(7)
-; GFX11-NEXT: v_lshl_or_b32 v1, v4, 8, v3
-; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 16, v5
-; GFX11-NEXT: s_waitcnt vmcnt(5)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: s_waitcnt vmcnt(4)
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX11-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-NEXT: v_and_or_b32 v3, 0xffff, s4, v3
-; GFX11-NEXT: s_waitcnt vmcnt(3)
-; GFX11-NEXT: v_or3_b32 v4, v5, v8, v4
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX11-NEXT: v_lshl_or_b32 v1, v9, 24, v1
-; GFX11-NEXT: v_mul_hi_u32 v3, 0xaaaaaab, v3
-; GFX11-NEXT: v_lshl_add_u32 v0, v0, 3, v2
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v10
-; GFX11-NEXT: v_lshrrev_b32_e32 v4, 1, v4
-; GFX11-NEXT: v_alignbit_b32 v1, v8, v1, 1
-; GFX11-NEXT: v_alignbit_b32 v5, s0, v5, 1
-; GFX11-NEXT: v_not_b32_e32 v0, v0
-; GFX11-NEXT: v_and_or_b32 v2, 0xffff, s0, v2
-; GFX11-NEXT: v_lshl_add_u32 v3, v3, 3, s4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_alignbit_b32 v0, v4, v1, v0.l
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, 1, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_not_b32_e32 v2, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_alignbit_b32 v1, v1, v5, v2.l
-; GFX11-NEXT: s_clause 0x4
-; GFX11-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
-; GFX11-NEXT: scratch_store_b8 off, v3, off offset:28
-; GFX11-NEXT: scratch_store_b8 off, v0, off offset:27
-; GFX11-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
-; GFX11-NEXT: scratch_store_b16 off, v1, off offset:24
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: scratch_load_b32 v0, off, off offset:24
-; GFX11-NEXT: scratch_load_u16 v1, off, off offset:28
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_lshr_b32 s11, s3, 8
+; GFX11-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: s_or_b32 s1, s8, s1
+; GFX11-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-NEXT: s_lshr_b32 s10, s2, 24
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-NEXT: s_and_b32 s8, s8, 0xff
+; GFX11-NEXT: s_or_b32 s3, s10, s3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_lshr_b32 s10, s4, 8
+; GFX11-NEXT: s_or_b32 s0, s0, s6
+; GFX11-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-NEXT: v_readfirstlane_b32 s13, v0
+; GFX11-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: s_lshl_b32 s8, s8, 8
+; GFX11-NEXT: s_and_b32 s10, s10, 0xff
+; GFX11-NEXT: s_or_b32 s2, s2, s8
+; GFX11-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-NEXT: s_and_b32 s9, s11, 0xff
+; GFX11-NEXT: s_lshr_b32 s11, s4, 16
+; GFX11-NEXT: s_lshr_b32 s12, s4, 24
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_lshl_b32 s10, s10, 8
+; GFX11-NEXT: s_and_b32 s11, s11, 0xff
+; GFX11-NEXT: s_or_b32 s4, s4, s10
+; GFX11-NEXT: s_mul_i32 s10, s13, 0xffffffe8
+; GFX11-NEXT: s_pack_ll_b32_b16 s4, s4, s11
+; GFX11-NEXT: s_mul_hi_u32 s10, s13, s10
+; GFX11-NEXT: s_lshr_b32 s14, s5, 8
+; GFX11-NEXT: s_add_i32 s13, s13, s10
+; GFX11-NEXT: s_and_b32 s5, s5, 0xff
+; GFX11-NEXT: s_mul_hi_u32 s10, s4, s13
+; GFX11-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-NEXT: s_mulk_i32 s10, 0xffe8
+; GFX11-NEXT: s_or_b32 s5, s12, s5
+; GFX11-NEXT: s_add_i32 s4, s4, s10
+; GFX11-NEXT: s_and_b32 s10, s14, 0xff
+; GFX11-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_cselect_b32 s11, 1, 0
+; GFX11-NEXT: s_sub_i32 s12, s4, 24
+; GFX11-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s8
+; GFX11-NEXT: s_cselect_b32 s4, s12, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s5, s5, s10
+; GFX11-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s3, s9
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_sub_i32 s8, s4, 24
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s7
+; GFX11-NEXT: s_cselect_b32 s4, s8, s4
+; GFX11-NEXT: s_lshr_b32 s2, s2, 1
+; GFX11-NEXT: s_sub_i32 s6, 23, s4
+; GFX11-NEXT: s_lshl_b32 s0, s0, s4
+; GFX11-NEXT: s_mul_hi_u32 s4, s5, s13
+; GFX11-NEXT: s_lshr_b32 s2, s2, s6
+; GFX11-NEXT: s_mulk_i32 s4, 0xffe8
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s5, s5, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s5, 24
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s5, 24
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_sub_i32 s5, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-NEXT: s_lshr_b32 s3, s3, 1
+; GFX11-NEXT: s_sub_i32 s4, 23, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, s2
+; GFX11-NEXT: s_lshr_b32 s2, s3, s4
+; GFX11-NEXT: s_and_b32 s3, s0, 0xff
+; GFX11-NEXT: s_or_b32 s1, s1, s2
+; GFX11-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX11-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX11-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-NEXT: s_or_b32 s2, s3, s2
+; GFX11-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-NEXT: s_or_b32 s0, s2, s0
+; GFX11-NEXT: s_lshl_b32 s2, s3, 24
+; GFX11-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX11-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshl_v2i24:
; GFX9: ; %bb.0:
@@ -3783,39 +3941,47 @@ define amdgpu_ps half @v_fshl_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
define amdgpu_ps i32 @s_fshl_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
; GFX6-LABEL: s_fshl_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_mov_b32 s4, s1
-; GFX6-NEXT: s_lshr_b32 s5, s0, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s5, s2, 15
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_lshr_b32 s4, s2, 16
+; GFX6-NEXT: s_andn2_b32 s2, 15, s2
+; GFX6-NEXT: s_lshl_b32 s0, s0, s5
+; GFX6-NEXT: s_bfe_u32 s5, s1, 0xf0001
+; GFX6-NEXT: s_lshr_b32 s2, s5, s2
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s4, 15
+; GFX6-NEXT: s_andn2_b32 s4, 15, s4
+; GFX6-NEXT: s_lshr_b32 s1, s1, 17
+; GFX6-NEXT: s_lshl_b32 s2, s3, s2
+; GFX6-NEXT: s_lshr_b32 s1, s1, s4
+; GFX6-NEXT: s_or_b32 s1, s2, s1
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: s_and_b32 s1, s2, 15
-; GFX6-NEXT: s_lshl_b32 s0, s0, s1
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], 16
-; GFX6-NEXT: s_bfe_u32 s1, s2, 0x40010
-; GFX6-NEXT: s_lshl_b32 s1, s4, s1
-; GFX6-NEXT: s_lshr_b32 s1, s1, 16
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 16
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_andn2_b32 s3, 15, s2
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0xf0001
-; GFX8-NEXT: s_lshr_b32 s3, s4, s3
-; GFX8-NEXT: s_and_b32 s4, s2, 15
-; GFX8-NEXT: s_lshl_b32 s4, s0, s4
-; GFX8-NEXT: s_or_b32 s3, s4, s3
+; GFX8-NEXT: s_and_b32 s5, s2, 15
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: s_lshl_b32 s0, s0, s5
+; GFX8-NEXT: s_and_b32 s5, 0xffff, s1
; GFX8-NEXT: s_lshr_b32 s4, s2, 16
-; GFX8-NEXT: s_lshr_b32 s1, s1, 17
+; GFX8-NEXT: s_andn2_b32 s2, 15, s2
+; GFX8-NEXT: s_lshr_b32 s5, s5, 1
+; GFX8-NEXT: s_lshr_b32 s2, s5, s2
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, s4, 15
; GFX8-NEXT: s_andn2_b32 s4, 15, s4
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40010
+; GFX8-NEXT: s_lshr_b32 s1, s1, 17
+; GFX8-NEXT: s_lshl_b32 s2, s3, s2
; GFX8-NEXT: s_lshr_b32 s1, s1, s4
-; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: s_or_b32 s2, s2, s1
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v2i16:
@@ -3894,34 +4060,50 @@ define <2 x i16> @v_fshl_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX6-LABEL: v_fshl_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v5, 15, v2
+; GFX6-NEXT: v_xor_b32_e32 v2, -1, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX6-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX6-NEXT: v_alignbit_b32 v1, v3, v1, 16
-; GFX6-NEXT: v_bfe_u32 v2, v2, 16, 4
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, v2, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v5, v0
+; GFX6-NEXT: v_bfe_u32 v5, v1, 1, 15
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v5
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v4
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 17, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, -1
-; GFX8-NEXT: v_xor_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 17, v1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v4, v4, v5
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 1, v1
; GFX8-NEXT: v_xor_b32_e32 v4, -1, v2
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, v4, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v2, v0
+; GFX8-NEXT: v_and_b32_e32 v3, 15, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshrrev_b16_e32 v5, 1, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, v3, v0
+; GFX8-NEXT: v_lshrrev_b16_e32 v4, v4, v5
+; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_mov_b32_e32 v4, 15
+; GFX8-NEXT: v_mov_b32_e32 v5, -1
+; GFX8-NEXT: v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v1, 17, v1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshrrev_b16_e32 v1, v2, v1
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_v2i16:
@@ -3968,24 +4150,29 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v1, 24
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 28
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_bfe_u32 v3, v1, 1, 15
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 11, v3
+; GFX6-NEXT: v_alignbit_b32 v1, v2, v1, 24
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v2i16_4_8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s4, 0xc0c0207
-; GFX8-NEXT: v_perm_b32 v2, v1, v0, s4
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 12, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX8-NEXT: v_lshrrev_b16_e32 v3, 12, v1
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: v_lshlrev_b16_e32 v0, 4, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_v2i16_4_8:
@@ -4019,36 +4206,51 @@ define <2 x i16> @v_fshl_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> %amt) {
; GFX6-LABEL: v_fshl_v2i16_ssv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_mov_b32 s2, s1
-; GFX6-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: v_and_b32_e32 v1, 15, v0
-; GFX6-NEXT: v_lshl_b32_e32 v1, s0, v1
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[2:3], 16
-; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 4
-; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v0
+; GFX6-NEXT: v_xor_b32_e32 v0, -1, v0
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT: v_lshl_b32_e32 v2, s0, v2
+; GFX6-NEXT: s_bfe_u32 s0, s1, 0xf0001
+; GFX6-NEXT: v_lshr_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v1
+; GFX6-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX6-NEXT: s_lshr_b32 s0, s1, 17
+; GFX6-NEXT: v_lshl_b32_e32 v2, s2, v2
+; GFX6-NEXT: v_lshr_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_v2i16_ssv:
; GFX8: ; %bb.0:
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v0
; GFX8-NEXT: s_lshr_b32 s2, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v2, -1
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_xor_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_lshr_b32 s2, s1, 17
-; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_e64 v2, v2, s2
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e64 v2, v0, s0
-; GFX8-NEXT: v_xor_b32_e32 v0, -1, v0
-; GFX8-NEXT: s_bfe_u32 s0, s1, 0xf0001
+; GFX8-NEXT: v_xor_b32_e32 v2, -1, v0
+; GFX8-NEXT: v_lshlrev_b16_e64 v1, v1, s0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s1
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT: s_lshr_b32 s0, s0, 1
+; GFX8-NEXT: v_lshrrev_b16_e64 v2, v2, s0
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, 15
+; GFX8-NEXT: v_mov_b32_e32 v3, -1
+; GFX8-NEXT: v_and_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: s_lshr_b32 s0, s1, 17
+; GFX8-NEXT: v_lshlrev_b16_e64 v2, v2, s2
; GFX8-NEXT: v_lshrrev_b16_e64 v0, v0, s0
; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshl_v2i16_ssv:
@@ -4103,36 +4305,44 @@ define amdgpu_ps float @v_fshl_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <2 x i16> inreg %amt) {
; GFX6-LABEL: v_fshl_v2i16_svs:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: s_and_b32 s4, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: v_bfe_u32 v1, v0, 1, 15
; GFX6-NEXT: s_lshr_b32 s2, s0, 16
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, s4
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, s1, v1
; GFX6-NEXT: v_or_b32_e32 v1, s0, v1
-; GFX6-NEXT: s_and_b32 s0, s1, 15
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX6-NEXT: v_alignbit_b32 v0, s2, v0, 16
-; GFX6-NEXT: s_bfe_u32 s0, s1, 0x40010
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 16
+; GFX6-NEXT: s_and_b32 s0, s3, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s3
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 17, v0
+; GFX6-NEXT: s_lshl_b32 s0, s2, s0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s1, v0
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_v2i16_svs:
; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_and_b32 s4, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: v_lshrrev_b16_e32 v1, 1, v0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, s2, v1
-; GFX8-NEXT: s_and_b32 s2, s1, 15
-; GFX8-NEXT: s_lshl_b32 s2, s0, s2
-; GFX8-NEXT: v_or_b32_e32 v1, s2, v1
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x40010
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshl_b32 s0, s0, s4
+; GFX8-NEXT: v_lshrrev_b16_e32 v1, s1, v1
+; GFX8-NEXT: v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT: s_and_b32 s0, s3, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s3
; GFX8-NEXT: v_lshrrev_b32_e32 v0, 17, v0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s2
-; GFX8-NEXT: s_lshl_b32 s0, s0, s1
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, s2, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: s_lshl_b32 s0, s2, s0
+; GFX8-NEXT: v_lshrrev_b16_e32 v0, s1, v0
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4185,36 +4395,45 @@ define amdgpu_ps float @v_fshl_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
; GFX6-LABEL: v_fshl_v2i16_vss:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_and_b32 s3, s1, 15
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: s_and_b32 s2, s0, 0xffff
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_alignbit_b32 v1, v1, s0, 16
-; GFX6-NEXT: s_bfe_u32 s0, s1, 0x40010
-; GFX6-NEXT: v_or_b32_e32 v0, s2, v0
-; GFX6-NEXT: s_and_b32 s2, s1, 15
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, s0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, s2, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 16
+; GFX6-NEXT: s_lshr_b32 s2, s1, 16
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s3, v0
+; GFX6-NEXT: s_bfe_u32 s3, s0, 0xf0001
+; GFX6-NEXT: s_lshr_b32 s1, s3, s1
+; GFX6-NEXT: v_or_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_and_b32 s1, s2, 15
+; GFX6-NEXT: s_andn2_b32 s2, 15, s2
+; GFX6-NEXT: s_lshr_b32 s0, s0, 17
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, s1, v1
+; GFX6-NEXT: s_lshr_b32 s0, s0, s2
+; GFX6-NEXT: v_or_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshl_v2i16_vss:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s2, s1, 15
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, s2, v0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
-; GFX8-NEXT: s_bfe_u32 s3, s0, 0xf0001
-; GFX8-NEXT: s_lshr_b32 s2, s3, s2
-; GFX8-NEXT: v_or_b32_e32 v1, s2, v1
+; GFX8-NEXT: s_and_b32 s3, s1, 15
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, s3, v0
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s0
; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 17
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
+; GFX8-NEXT: s_lshr_b32 s3, s3, 1
+; GFX8-NEXT: s_lshr_b32 s1, s3, s1
+; GFX8-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX8-NEXT: s_and_b32 s1, s2, 15
; GFX8-NEXT: s_andn2_b32 s2, 15, s2
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x40010
-; GFX8-NEXT: s_lshr_b32 s0, s0, s2
; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: s_lshr_b32 s0, s0, 17
; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: s_lshr_b32 s0, s0, s2
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4282,68 +4501,62 @@ define amdgpu_ps float @v_fshl_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
define amdgpu_ps i48 @s_fshl_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <3 x i16> inreg %amt) {
; GFX6-LABEL: s_fshl_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_mov_b32 s6, s3
-; GFX6-NEXT: s_lshr_b32 s7, s1, 16
-; GFX6-NEXT: s_and_b32 s8, s6, 0xffff
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_or_b32 s1, s1, s8
-; GFX6-NEXT: s_and_b32 s8, s5, 15
-; GFX6-NEXT: s_lshl_b32 s8, s1, s8
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
-; GFX6-NEXT: s_bfe_u32 s1, s5, 0x40010
-; GFX6-NEXT: s_lshl_b32 s1, s6, s1
-; GFX6-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-NEXT: s_lshr_b32 s9, s1, 16
-; GFX6-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: s_and_b32 s1, s4, 15
-; GFX6-NEXT: s_lshl_b32 s0, s0, s1
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], 16
-; GFX6-NEXT: s_bfe_u32 s1, s4, 0x40010
-; GFX6-NEXT: s_lshl_b32 s1, s2, s1
-; GFX6-NEXT: s_lshr_b32 s1, s1, 16
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[8:9], 16
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 16
-; GFX6-NEXT: s_mov_b32 s1, s6
+; GFX6-NEXT: s_and_b32 s8, s4, 15
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_andn2_b32 s4, 15, s4
+; GFX6-NEXT: s_lshl_b32 s0, s0, s8
+; GFX6-NEXT: s_bfe_u32 s8, s2, 0xf0001
+; GFX6-NEXT: s_lshr_b32 s4, s8, s4
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 15
+; GFX6-NEXT: s_andn2_b32 s7, 15, s7
+; GFX6-NEXT: s_lshr_b32 s2, s2, 17
+; GFX6-NEXT: s_lshl_b32 s4, s6, s4
+; GFX6-NEXT: s_lshr_b32 s2, s2, s7
+; GFX6-NEXT: s_or_b32 s2, s4, s2
+; GFX6-NEXT: s_and_b32 s4, s5, 15
+; GFX6-NEXT: s_andn2_b32 s5, 15, s5
+; GFX6-NEXT: s_bfe_u32 s3, s3, 0xf0001
+; GFX6-NEXT: s_lshl_b32 s1, s1, s4
+; GFX6-NEXT: s_lshr_b32 s3, s3, s5
+; GFX6-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX6-NEXT: s_or_b32 s1, s1, s3
+; GFX6-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_v3i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_andn2_b32 s6, 15, s5
-; GFX8-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX8-NEXT: s_lshr_b32 s6, s7, s6
-; GFX8-NEXT: s_and_b32 s7, s5, 15
-; GFX8-NEXT: s_lshl_b32 s7, s1, s7
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshr_b32 s7, s5, 16
-; GFX8-NEXT: s_lshr_b32 s3, s3, 17
+; GFX8-NEXT: s_and_b32 s8, s4, 15
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: s_lshl_b32 s0, s0, s8
+; GFX8-NEXT: s_and_b32 s8, 0xffff, s2
+; GFX8-NEXT: s_lshr_b32 s7, s4, 16
+; GFX8-NEXT: s_andn2_b32 s4, 15, s4
+; GFX8-NEXT: s_lshr_b32 s8, s8, 1
+; GFX8-NEXT: s_lshr_b32 s4, s8, s4
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, s7, 15
; GFX8-NEXT: s_andn2_b32 s7, 15, s7
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX8-NEXT: s_lshr_b32 s3, s3, s7
-; GFX8-NEXT: s_lshl_b32 s1, s1, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s3
-; GFX8-NEXT: s_andn2_b32 s3, 15, s4
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0xf0001
-; GFX8-NEXT: s_lshr_b32 s3, s5, s3
-; GFX8-NEXT: s_and_b32 s5, s4, 15
-; GFX8-NEXT: s_lshl_b32 s5, s0, s5
-; GFX8-NEXT: s_or_b32 s3, s5, s3
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
; GFX8-NEXT: s_lshr_b32 s2, s2, 17
+; GFX8-NEXT: s_lshl_b32 s4, s6, s4
+; GFX8-NEXT: s_lshr_b32 s2, s2, s7
+; GFX8-NEXT: s_or_b32 s4, s4, s2
+; GFX8-NEXT: s_and_b32 s2, s5, 15
+; GFX8-NEXT: s_lshl_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
; GFX8-NEXT: s_andn2_b32 s5, 15, s5
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX8-NEXT: s_lshr_b32 s2, s2, 1
; GFX8-NEXT: s_lshr_b32 s2, s2, s5
-; GFX8-NEXT: s_lshl_b32 s0, s0, s4
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s4
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
; GFX8-NEXT: s_or_b32 s0, s0, s2
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s1, s6, s1
-; GFX8-NEXT: s_or_b32 s0, s3, s0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v3i16:
@@ -4534,45 +4747,66 @@ define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX6-LABEL: v_fshl_v3i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v8, 15, v4
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v7, 0xffff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v7
-; GFX6-NEXT: v_and_b32_e32 v7, 15, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v6, v2, 16
-; GFX6-NEXT: v_bfe_u32 v4, v4, 16, 4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, v7, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_alignbit_b32 v0, v2, v0, 16
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_and_b32_e32 v2, 15, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, v2, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v8, v0
+; GFX6-NEXT: v_bfe_u32 v8, v2, 1, 15
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v4, v8
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v7
+; GFX6-NEXT: v_xor_b32_e32 v7, -1, v7
+; GFX6-NEXT: v_and_b32_e32 v7, 15, v7
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 17, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v7, v2
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v5
+; GFX6-NEXT: v_xor_b32_e32 v5, -1, v5
+; GFX6-NEXT: v_and_b32_e32 v5, 15, v5
+; GFX6-NEXT: v_bfe_u32 v3, v3, 1, 15
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v3
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v3i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, -1
-; GFX8-NEXT: v_xor_b32_sdwa v7, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 17, v2
-; GFX8-NEXT: v_lshlrev_b16_sdwa v6, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_xor_b32_e32 v7, -1, v4
+; GFX8-NEXT: v_and_b32_e32 v6, 15, v4
+; GFX8-NEXT: v_and_b32_e32 v7, 15, v7
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 1, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, v6, v0
; GFX8-NEXT: v_lshrrev_b16_e32 v7, v7, v8
-; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, 1, v3
-; GFX8-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, v7, v3
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, 1, v2
-; GFX8-NEXT: v_xor_b32_e32 v3, -1, v4
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, v3, v2
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX8-NEXT: v_mov_b32_e32 v7, 15
+; GFX8-NEXT: v_mov_b32_e32 v8, -1
+; GFX8-NEXT: v_and_b32_sdwa v7, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 17, v2
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v4, v2
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v5
+; GFX8-NEXT: v_xor_b32_e32 v4, -1, v5
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, v2, v1
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, 1, v3
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v4, v2
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_v3i16:
@@ -4639,68 +4873,84 @@ define <3 x half> @v_fshl_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
define amdgpu_ps <2 x i32> @s_fshl_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs, <4 x i16> inreg %amt) {
; GFX6-LABEL: s_fshl_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_mov_b32 s6, s3
+; GFX6-NEXT: s_and_b32 s10, s4, 15
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshr_b32 s8, s4, 16
+; GFX6-NEXT: s_andn2_b32 s4, 15, s4
+; GFX6-NEXT: s_lshl_b32 s0, s0, s10
+; GFX6-NEXT: s_bfe_u32 s10, s2, 0xf0001
+; GFX6-NEXT: s_lshr_b32 s4, s10, s4
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_and_b32 s4, s8, 15
+; GFX6-NEXT: s_andn2_b32 s8, 15, s8
+; GFX6-NEXT: s_lshr_b32 s2, s2, 17
+; GFX6-NEXT: s_lshl_b32 s4, s6, s4
+; GFX6-NEXT: s_lshr_b32 s2, s2, s8
+; GFX6-NEXT: s_or_b32 s2, s4, s2
+; GFX6-NEXT: s_and_b32 s4, s5, 15
; GFX6-NEXT: s_lshr_b32 s7, s1, 16
-; GFX6-NEXT: s_and_b32 s8, s6, 0xffff
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_or_b32 s1, s1, s8
-; GFX6-NEXT: s_and_b32 s8, s5, 15
-; GFX6-NEXT: s_lshl_b32 s8, s1, s8
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[6:7], 16
-; GFX6-NEXT: s_bfe_u32 s1, s5, 0x40010
-; GFX6-NEXT: s_lshl_b32 s1, s6, s1
-; GFX6-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-NEXT: s_lshr_b32 s9, s1, 16
-; GFX6-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s0, s0, s1
-; GFX6-NEXT: s_and_b32 s1, s4, 15
-; GFX6-NEXT: s_lshl_b32 s0, s0, s1
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[2:3], 16
-; GFX6-NEXT: s_bfe_u32 s1, s4, 0x40010
-; GFX6-NEXT: s_lshl_b32 s1, s2, s1
-; GFX6-NEXT: s_lshr_b32 s1, s1, 16
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[8:9], 16
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], 16
-; GFX6-NEXT: s_mov_b32 s1, s6
+; GFX6-NEXT: s_lshr_b32 s9, s5, 16
+; GFX6-NEXT: s_andn2_b32 s5, 15, s5
+; GFX6-NEXT: s_lshl_b32 s1, s1, s4
+; GFX6-NEXT: s_bfe_u32 s4, s3, 0xf0001
+; GFX6-NEXT: s_lshr_b32 s4, s4, s5
+; GFX6-NEXT: s_or_b32 s1, s1, s4
+; GFX6-NEXT: s_and_b32 s4, s9, 15
+; GFX6-NEXT: s_andn2_b32 s5, 15, s9
+; GFX6-NEXT: s_lshr_b32 s3, s3, 17
+; GFX6-NEXT: s_lshl_b32 s4, s7, s4
+; GFX6-NEXT: s_lshr_b32 s3, s3, s5
+; GFX6-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX6-NEXT: s_or_b32 s3, s4, s3
+; GFX6-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s2
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshl_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_andn2_b32 s6, 15, s5
-; GFX8-NEXT: s_bfe_u32 s7, s3, 0xf0001
-; GFX8-NEXT: s_lshr_b32 s6, s7, s6
-; GFX8-NEXT: s_and_b32 s7, s5, 15
-; GFX8-NEXT: s_lshl_b32 s7, s1, s7
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshr_b32 s7, s5, 16
-; GFX8-NEXT: s_lshr_b32 s3, s3, 17
-; GFX8-NEXT: s_andn2_b32 s7, 15, s7
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX8-NEXT: s_lshr_b32 s3, s3, s7
-; GFX8-NEXT: s_lshl_b32 s1, s1, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s3
-; GFX8-NEXT: s_andn2_b32 s3, 15, s4
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0xf0001
-; GFX8-NEXT: s_lshr_b32 s3, s5, s3
-; GFX8-NEXT: s_and_b32 s5, s4, 15
-; GFX8-NEXT: s_lshl_b32 s5, s0, s5
-; GFX8-NEXT: s_or_b32 s3, s5, s3
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
+; GFX8-NEXT: s_and_b32 s10, s4, 15
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: s_lshl_b32 s0, s0, s10
+; GFX8-NEXT: s_and_b32 s10, 0xffff, s2
+; GFX8-NEXT: s_lshr_b32 s8, s4, 16
+; GFX8-NEXT: s_andn2_b32 s4, 15, s4
+; GFX8-NEXT: s_lshr_b32 s10, s10, 1
+; GFX8-NEXT: s_lshr_b32 s4, s10, s4
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, s8, 15
+; GFX8-NEXT: s_andn2_b32 s8, 15, s8
; GFX8-NEXT: s_lshr_b32 s2, s2, 17
+; GFX8-NEXT: s_lshl_b32 s4, s6, s4
+; GFX8-NEXT: s_lshr_b32 s2, s2, s8
+; GFX8-NEXT: s_or_b32 s4, s4, s2
+; GFX8-NEXT: s_and_b32 s2, s5, 15
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: s_lshl_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX8-NEXT: s_lshr_b32 s9, s5, 16
; GFX8-NEXT: s_andn2_b32 s5, 15, s5
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_bfe_u32 s4, s4, 0x40010
+; GFX8-NEXT: s_lshr_b32 s2, s2, 1
; GFX8-NEXT: s_lshr_b32 s2, s2, s5
-; GFX8-NEXT: s_lshl_b32 s0, s0, s4
-; GFX8-NEXT: s_or_b32 s0, s0, s2
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s1, s6, s1
-; GFX8-NEXT: s_or_b32 s0, s3, s0
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, s9, 15
+; GFX8-NEXT: s_andn2_b32 s5, 15, s9
+; GFX8-NEXT: s_lshr_b32 s3, s3, 17
+; GFX8-NEXT: s_lshl_b32 s2, s7, s2
+; GFX8-NEXT: s_lshr_b32 s3, s3, s5
+; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s4
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshl_v4i16:
@@ -4835,56 +5085,87 @@ define <4 x half> @v_fshl_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX6-LABEL: v_fshl_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX6-NEXT: v_and_b32_e32 v8, 15, v4
-; GFX6-NEXT: v_alignbit_b32 v2, v7, v2, 16
-; GFX6-NEXT: v_bfe_u32 v4, v4, 16, 4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, v8, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v0, v2, v0, 16
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v10, 15, v4
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v10, v0
+; GFX6-NEXT: v_bfe_u32 v10, v2, 1, 15
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v4, v10
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v8
+; GFX6-NEXT: v_xor_b32_e32 v8, -1, v8
+; GFX6-NEXT: v_and_b32_e32 v8, 15, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 17, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v8, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v5
+; GFX6-NEXT: v_xor_b32_e32 v5, -1, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v5, 15, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_bfe_u32 v4, v3, 1, 15
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v5, v4
+; GFX6-NEXT: v_xor_b32_e32 v5, -1, v9
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v9
+; GFX6-NEXT: v_and_b32_e32 v5, 15, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 17, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v7
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v5, v3
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_and_b32_e32 v2, 15, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, v2, v1
-; GFX6-NEXT: v_alignbit_b32 v2, v6, v3, 16
-; GFX6-NEXT: v_bfe_u32 v3, v5, 16, 4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_alignbit_b32 v1, v2, v1, 16
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshl_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, -1
-; GFX8-NEXT: v_xor_b32_sdwa v8, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 17, v3
-; GFX8-NEXT: v_lshlrev_b16_sdwa v6, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v8, v8, v9
-; GFX8-NEXT: v_xor_b32_sdwa v7, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 17, v2
-; GFX8-NEXT: v_or_b32_sdwa v6, v6, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_sdwa v8, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v7, v7, v9
-; GFX8-NEXT: v_or_b32_sdwa v7, v8, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, 1, v3
-; GFX8-NEXT: v_xor_b32_e32 v8, -1, v5
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, v8, v3
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, 1, v2
-; GFX8-NEXT: v_xor_b32_e32 v3, -1, v4
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, v3, v2
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v4, v0
+; GFX8-NEXT: v_xor_b32_e32 v7, -1, v4
+; GFX8-NEXT: v_and_b32_e32 v6, 15, v4
+; GFX8-NEXT: v_and_b32_e32 v7, 15, v7
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 1, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, v6, v0
+; GFX8-NEXT: v_lshrrev_b16_e32 v7, v7, v8
+; GFX8-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX8-NEXT: v_mov_b32_e32 v7, 15
+; GFX8-NEXT: v_mov_b32_e32 v9, -1
+; GFX8-NEXT: v_and_b32_sdwa v8, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 17, v2
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v4, v2
+; GFX8-NEXT: v_xor_b32_e32 v4, -1, v5
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v5
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, 1, v3
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, v2, v1
+; GFX8-NEXT: v_lshrrev_b16_e32 v4, v4, v8
+; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX8-NEXT: v_and_b32_sdwa v4, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v5, v5, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v5, 15, v5
+; GFX8-NEXT: v_lshrrev_b32_e32 v3, 17, v3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshrrev_b16_e32 v3, v5, v3
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshl_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
index bccd8958962da..9ab63e303c895 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fshr.ll
@@ -995,105 +995,165 @@ define i16 @v_fshr_v2i8(i16 %lhs.arg, i16 %rhs.arg, i16 %amt.arg) {
define amdgpu_ps i32 @s_fshr_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg, i32 inreg %amt.arg) {
; GFX6-LABEL: s_fshr_v4i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s6, s2, 24
-; GFX6-NEXT: s_mov_b32 s5, s0
-; GFX6-NEXT: s_or_b32 s6, s6, 24
-; GFX6-NEXT: s_lshr_b32 s4, s2, 16
-; GFX6-NEXT: s_lshr_b32 s9, s5, 24
-; GFX6-NEXT: s_and_b32 s8, s1, 0xff000000
-; GFX6-NEXT: s_and_b32 s6, s6, 31
-; GFX6-NEXT: s_lshr_b64 s[8:9], s[8:9], s6
-; GFX6-NEXT: s_lshl_b32 s6, s1, 8
-; GFX6-NEXT: s_or_b32 s4, s4, 24
-; GFX6-NEXT: s_lshr_b32 s7, s5, 16
-; GFX6-NEXT: s_and_b32 s6, s6, 0xff000000
-; GFX6-NEXT: s_and_b32 s4, s4, 31
-; GFX6-NEXT: s_lshr_b64 s[6:7], s[6:7], s4
-; GFX6-NEXT: s_and_b32 s4, s6, 0xff
-; GFX6-NEXT: s_lshr_b32 s0, s2, 8
-; GFX6-NEXT: s_lshl_b32 s8, s8, 24
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
-; GFX6-NEXT: s_or_b32 s2, s2, 24
-; GFX6-NEXT: s_or_b32 s6, s8, s4
-; GFX6-NEXT: s_lshl_b32 s4, s1, 24
-; GFX6-NEXT: s_and_b32 s2, s2, 31
+; GFX6-NEXT: s_lshr_b32 s3, s0, 8
+; GFX6-NEXT: s_lshr_b32 s4, s0, 16
+; GFX6-NEXT: s_lshr_b32 s5, s0, 24
+; GFX6-NEXT: s_lshr_b32 s7, s2, 8
+; GFX6-NEXT: s_lshr_b32 s8, s2, 16
+; GFX6-NEXT: s_lshr_b32 s9, s2, 24
+; GFX6-NEXT: s_and_b32 s10, s2, 7
+; GFX6-NEXT: s_andn2_b32 s2, 7, s2
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: s_lshl_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s1, 0xff
+; GFX6-NEXT: s_lshr_b32 s2, s2, s10
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s7, 7
+; GFX6-NEXT: s_andn2_b32 s7, 7, s7
+; GFX6-NEXT: s_lshl_b32 s3, s3, 1
+; GFX6-NEXT: s_lshl_b32 s3, s3, s7
+; GFX6-NEXT: s_bfe_u32 s7, s1, 0x80008
+; GFX6-NEXT: s_lshr_b32 s2, s7, s2
+; GFX6-NEXT: s_lshr_b32 s6, s1, 24
+; GFX6-NEXT: s_or_b32 s3, s3, s2
+; GFX6-NEXT: s_and_b32 s2, s8, 7
+; GFX6-NEXT: s_andn2_b32 s7, 7, s8
+; GFX6-NEXT: s_lshl_b32 s4, s4, 1
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX6-NEXT: s_lshl_b32 s4, s4, s7
+; GFX6-NEXT: s_lshr_b32 s1, s1, s2
+; GFX6-NEXT: s_or_b32 s4, s4, s1
+; GFX6-NEXT: s_and_b32 s1, s9, 7
+; GFX6-NEXT: s_andn2_b32 s2, 7, s9
+; GFX6-NEXT: s_lshl_b32 s5, s5, 1
+; GFX6-NEXT: s_lshl_b32 s2, s5, s2
+; GFX6-NEXT: s_lshr_b32 s1, s6, s1
+; GFX6-NEXT: s_or_b32 s2, s2, s1
+; GFX6-NEXT: s_and_b32 s1, s3, 0xff
+; GFX6-NEXT: s_and_b32 s0, s0, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s4, 0xff
; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_or_b32 s0, s0, 24
-; GFX6-NEXT: s_bfe_u32 s3, s5, 0x80008
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s2
-; GFX6-NEXT: s_and_b32 s2, s1, 0xff000000
-; GFX6-NEXT: s_and_b32 s0, s0, 31
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX6-NEXT: s_and_b32 s4, s4, 0xff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 8
-; GFX6-NEXT: s_or_b32 s0, s4, s0
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s0, s6
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s2, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_v4i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, 0xc0c0307
-; GFX8-NEXT: v_perm_b32 v1, s1, v0, v1
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xc0c0206
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, s0, v1
-; GFX8-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30010
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, s0, v2
-; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xc0c0105
-; GFX8-NEXT: v_perm_b32 v2, s1, v0, v2
-; GFX8-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX8-NEXT: v_mov_b32_e32 v3, 0xc0c0004
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, s0, v2
-; GFX8-NEXT: v_perm_b32 v0, s1, v0, v3
-; GFX8-NEXT: s_and_b32 s0, s2, 7
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_lshr_b32 s6, s1, 8
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: s_lshr_b32 s8, s1, 24
+; GFX8-NEXT: s_xor_b32 s12, s2, -1
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_lshr_b32 s3, s0, 8
+; GFX8-NEXT: s_lshr_b32 s4, s0, 16
+; GFX8-NEXT: s_lshr_b32 s5, s0, 24
+; GFX8-NEXT: s_lshr_b32 s9, s2, 8
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: s_lshr_b32 s11, s2, 24
+; GFX8-NEXT: s_lshl_b32 s0, s0, 1
+; GFX8-NEXT: s_and_b32 s12, s12, 7
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s12
+; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_xor_b32 s2, s9, -1
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s3, 1
+; GFX8-NEXT: s_and_b32 s2, s2, 7
+; GFX8-NEXT: s_and_b32 s3, s6, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, s9, 7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshr_b32 s2, s3, s2
+; GFX8-NEXT: s_xor_b32 s3, s10, -1
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_lshl_b32 s2, s4, 1
+; GFX8-NEXT: s_and_b32 s3, s3, 7
+; GFX8-NEXT: s_and_b32 s4, s7, 0xff
+; GFX8-NEXT: s_lshl_b32 s2, s2, s3
+; GFX8-NEXT: s_and_b32 s3, s10, 7
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_lshr_b32 s3, s4, s3
+; GFX8-NEXT: s_xor_b32 s4, s11, -1
+; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_lshl_b32 s3, s5, 1
+; GFX8-NEXT: s_and_b32 s4, s4, 7
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_lshl_b32 s3, s3, s4
+; GFX8-NEXT: s_and_b32 s4, s11, 7
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s4, s8, s4
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xff
+; GFX8-NEXT: s_or_b32 s3, s3, s4
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s3, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 24
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v4i8:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_mov_b32_e32 v0, 0xc0c0105
-; GFX11-NEXT: s_bfe_u32 s3, s2, 0x30008
-; GFX11-NEXT: v_mov_b32_e32 v2, 0xc0c0307
-; GFX11-NEXT: v_mov_b32_e32 v3, 0xc0c0206
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v0, s1, s0, v0
-; GFX11-NEXT: v_perm_b32 v2, s1, s0, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_perm_b32 v3, s1, s0, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v0, s3, v0
-; GFX11-NEXT: v_mov_b32_e32 v1, 0xc0c0004
-; GFX11-NEXT: s_and_b32 s3, s2, 7
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX11-NEXT: v_perm_b32 v1, s1, s0, v1
-; GFX11-NEXT: s_bfe_u32 s0, s2, 0x30018
-; GFX11-NEXT: s_bfe_u32 s1, s2, 0x30010
-; GFX11-NEXT: v_lshrrev_b32_e32 v2, s0, v2
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, s1, v3
-; GFX11-NEXT: v_lshrrev_b32_e32 v1, s3, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX11-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_or_b32_e32 v1, v3, v2
-; GFX11-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-NEXT: s_lshr_b32 s7, s1, 16
+; GFX11-NEXT: s_lshr_b32 s8, s1, 24
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshr_b32 s9, s2, 8
+; GFX11-NEXT: s_lshr_b32 s10, s2, 16
+; GFX11-NEXT: s_lshr_b32 s11, s2, 24
+; GFX11-NEXT: s_xor_b32 s12, s2, -1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX11-NEXT: s_lshr_b32 s3, s0, 8
+; GFX11-NEXT: s_lshr_b32 s1, s1, s2
+; GFX11-NEXT: s_xor_b32 s2, s9, -1
+; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s0, 24
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_and_b32 s12, s12, 7
+; GFX11-NEXT: s_lshl_b32 s3, s3, 1
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s9, s9, 7
+; GFX11-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX11-NEXT: s_lshl_b32 s0, s0, s12
+; GFX11-NEXT: s_lshl_b32 s2, s3, s2
+; GFX11-NEXT: s_lshr_b32 s3, s6, s9
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_or_b32 s1, s2, s3
+; GFX11-NEXT: s_xor_b32 s2, s10, -1
+; GFX11-NEXT: s_lshl_b32 s3, s4, 1
+; GFX11-NEXT: s_and_b32 s4, s7, 0xff
+; GFX11-NEXT: s_and_b32 s2, s2, 7
+; GFX11-NEXT: s_and_b32 s6, s10, 7
+; GFX11-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX11-NEXT: s_lshl_b32 s2, s3, s2
+; GFX11-NEXT: s_lshr_b32 s3, s4, s6
+; GFX11-NEXT: s_xor_b32 s4, s11, -1
+; GFX11-NEXT: s_lshl_b32 s5, s5, 1
+; GFX11-NEXT: s_and_b32 s4, s4, 7
+; GFX11-NEXT: s_and_b32 s6, s11, 7
+; GFX11-NEXT: s_lshl_b32 s4, s5, s4
+; GFX11-NEXT: s_lshr_b32 s5, s8, s6
+; GFX11-NEXT: s_or_b32 s2, s2, s3
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_or_b32 s3, s4, s5
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s1, s1, 8
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, 16
+; GFX11-NEXT: s_and_b32 s2, s3, 0xff
+; GFX11-NEXT: s_or_b32 s0, s0, s1
+; GFX11-NEXT: s_lshl_b32 s1, s2, 24
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_or_b32 s0, s0, s1
; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v4i8:
; GFX9: ; %bb.0:
@@ -1211,35 +1271,50 @@ define i32 @v_fshr_v4i8(i32 %lhs.arg, i32 %rhs.arg, i32 %amt.arg) {
; GFX6-LABEL: v_fshr_v4i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 24, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX6-NEXT: v_and_b32_e32 v9, 0xff000000, v1
-; GFX6-NEXT: v_or_b32_e32 v7, 24, v7
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX6-NEXT: v_alignbit_b32 v6, v6, v9, v7
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 8, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v5, 24, v5
-; GFX6-NEXT: v_and_b32_e32 v7, 0xff000000, v7
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v2
-; GFX6-NEXT: v_alignbit_b32 v4, v4, v7, v5
-; GFX6-NEXT: v_or_b32_e32 v2, 24, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 24, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_bfe_u32 v8, v0, 8, 8
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v5, v2
-; GFX6-NEXT: v_or_b32_e32 v2, 24, v3
-; GFX6-NEXT: v_and_b32_e32 v1, 0xff000000, v1
-; GFX6-NEXT: v_alignbit_b32 v1, v8, v1, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xff, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 8, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 24, v2
+; GFX6-NEXT: v_and_b32_e32 v10, 7, v2
+; GFX6-NEXT: v_bfi_b32 v2, v2, 0, 7
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v10, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 7, v7
+; GFX6-NEXT: v_bfi_b32 v7, v7, 0, 7
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 1, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v7, v3
+; GFX6-NEXT: v_bfe_u32 v7, v1, 8, 8
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v7
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 24, v1
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_and_b32_e32 v3, 7, v8
+; GFX6-NEXT: v_bfi_b32 v7, v8, 0, 7
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 1, v4
+; GFX6-NEXT: v_bfe_u32 v1, v1, 16, 8
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v7, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v3, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_and_b32_e32 v3, 7, v9
+; GFX6-NEXT: v_bfi_b32 v4, v9, 0, 7
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 1, v5
+; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v3, v6
; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 24, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: v_or_b32_e32 v4, v6, v4
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v4i8:
@@ -1779,282 +1854,365 @@ define i24 @v_fshr_i24(i24 %lhs, i24 %rhs, i24 %amt) {
define amdgpu_ps i48 @s_fshr_v2i24(i48 inreg %lhs.arg, i48 inreg %rhs.arg, i48 inreg %amt.arg) {
; GFX6-LABEL: s_fshr_v2i24:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_getpc_b64 s[8:9]
-; GFX6-NEXT: s_mov_b32 s8, s0
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX6-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_add_u32 s8, s8, s6
-; GFX6-NEXT: s_addc_u32 s9, s9, 0
-; GFX6-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:4
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_lshr_b32 s7, s1, 8
+; GFX6-NEXT: s_bfe_u32 s9, s0, 0x80008
+; GFX6-NEXT: s_and_b32 s1, s1, 0xff
; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s5
-; GFX6-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:20
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s3
-; GFX6-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_mov_b32_e32 v0, s2
-; GFX6-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:8
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:4
-; GFX6-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:5
-; GFX6-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:20
-; GFX6-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:21
-; GFX6-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:13
-; GFX6-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:12
-; GFX6-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:2
-; GFX6-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:18
-; GFX6-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:19
-; GFX6-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:10
-; GFX6-NEXT: buffer_load_ushort v10, off, s[8:11], 0 offset:8
-; GFX6-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:3
-; GFX6-NEXT: buffer_load_ushort v12, off, s[8:11], 0
-; GFX6-NEXT: buffer_load_ushort v13, off, s[8:11], 0 offset:16
-; GFX6-NEXT: buffer_load_ubyte v14, off, s[8:11], 0 offset:11
-; GFX6-NEXT: s_mov_b32 s0, 0xaaaaaab
-; GFX6-NEXT: s_waitcnt vmcnt(14)
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v0
-; GFX6-NEXT: s_waitcnt vmcnt(13)
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: s_waitcnt vmcnt(12)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX6-NEXT: s_waitcnt vmcnt(11)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: s_waitcnt vmcnt(10)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v4
-; GFX6-NEXT: s_waitcnt vmcnt(9)
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 8, v5
-; GFX6-NEXT: s_waitcnt vmcnt(8)
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX6-NEXT: s_waitcnt vmcnt(7)
-; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX6-NEXT: s_waitcnt vmcnt(6)
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v8
-; GFX6-NEXT: s_waitcnt vmcnt(5)
-; GFX6-NEXT: v_lshlrev_b32_e32 v9, 24, v9
-; GFX6-NEXT: s_waitcnt vmcnt(4)
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 8, v10
-; GFX6-NEXT: s_waitcnt vmcnt(3)
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v11
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: s_waitcnt vmcnt(1)
-; GFX6-NEXT: v_or_b32_e32 v7, v13, v7
-; GFX6-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX6-NEXT: v_mul_hi_u32 v2, v7, s0
-; GFX6-NEXT: v_mul_hi_u32 v1, v1, s0
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_or_b32_e32 v5, v5, v14
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v5
-; GFX6-NEXT: v_mul_u32_u24_e32 v2, 24, v2
-; GFX6-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v13, v2
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v8, v1
-; GFX6-NEXT: v_or_b32_e32 v6, v12, v6
-; GFX6-NEXT: v_or_b32_e32 v9, v9, v10
-; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v2
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, 8, v1
-; GFX6-NEXT: v_alignbit_b32 v2, v6, v9, v2
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v3, v1
-; GFX6-NEXT: buffer_store_short v2, off, s[8:11], 0 offset:24
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v2
-; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:27
-; GFX6-NEXT: buffer_store_byte v1, off, s[8:11], 0 offset:26
-; GFX6-NEXT: buffer_store_byte v2, off, s[8:11], 0 offset:29
-; GFX6-NEXT: buffer_store_byte v3, off, s[8:11], 0 offset:28
-; GFX6-NEXT: s_waitcnt expcnt(3)
-; GFX6-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX6-NEXT: s_waitcnt expcnt(2)
-; GFX6-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX6-NEXT: s_waitcnt vmcnt(1)
+; GFX6-NEXT: s_and_b32 s8, s0, 0xff
+; GFX6-NEXT: s_lshl_b32 s9, s9, 8
+; GFX6-NEXT: v_alignbit_b32 v0, s1, v0, 24
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_or_b32 s8, s8, s9
; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_readfirstlane_b32 s1, v1
-; GFX6-NEXT: s_waitcnt expcnt(0)
+; GFX6-NEXT: s_lshr_b32 s9, s3, 8
+; GFX6-NEXT: s_and_b32 s3, s3, 0xff
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_alignbit_b32 v0, s3, v0, 24
+; GFX6-NEXT: s_and_b32 s1, s7, 0xff
+; GFX6-NEXT: s_lshr_b32 s7, s2, 16
+; GFX6-NEXT: s_and_b32 s10, s2, 0xff
+; GFX6-NEXT: s_bfe_u32 s11, s2, 0x80008
+; GFX6-NEXT: v_readfirstlane_b32 s2, v0
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s3, s9, 0xff
+; GFX6-NEXT: s_lshl_b32 s11, s11, 8
+; GFX6-NEXT: s_and_b32 s7, s7, 0xff
+; GFX6-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX6-NEXT: s_or_b32 s10, s10, s11
+; GFX6-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX6-NEXT: s_and_b32 s10, 0xffff, s10
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s3
+; GFX6-NEXT: s_lshr_b32 s3, s4, 16
+; GFX6-NEXT: s_bfe_u32 s11, s4, 0x80008
+; GFX6-NEXT: s_or_b32 s7, s10, s7
+; GFX6-NEXT: s_and_b32 s10, s4, 0xff
+; GFX6-NEXT: s_lshl_b32 s11, s11, 8
+; GFX6-NEXT: s_and_b32 s3, s3, 0xff
+; GFX6-NEXT: s_or_b32 s10, s10, s11
+; GFX6-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX6-NEXT: s_and_b32 s10, 0xffff, s10
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: v_mov_b32_e32 v1, s4
+; GFX6-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-NEXT: s_or_b32 s3, s10, s3
+; GFX6-NEXT: s_mul_i32 s10, s4, 0xffffffe8
+; GFX6-NEXT: v_mul_hi_u32 v0, v0, s10
+; GFX6-NEXT: s_lshr_b32 s9, s5, 8
+; GFX6-NEXT: s_and_b32 s5, s5, 0xff
+; GFX6-NEXT: s_and_b32 s9, s9, 0xff
+; GFX6-NEXT: v_readfirstlane_b32 s10, v0
+; GFX6-NEXT: s_add_i32 s4, s4, s10
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT: v_alignbit_b32 v1, s5, v1, 24
+; GFX6-NEXT: s_and_b32 s9, 0xffff, s9
+; GFX6-NEXT: v_readfirstlane_b32 s5, v1
+; GFX6-NEXT: s_lshl_b32 s9, s9, 16
+; GFX6-NEXT: s_or_b32 s5, s5, s9
+; GFX6-NEXT: v_readfirstlane_b32 s9, v0
+; GFX6-NEXT: s_and_b32 s6, s6, 0xff
+; GFX6-NEXT: s_mulk_i32 s9, 0xffe8
+; GFX6-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX6-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_add_i32 s3, s3, s9
+; GFX6-NEXT: s_cmp_ge_u32 s3, 24
+; GFX6-NEXT: s_cselect_b32 s9, 1, 0
+; GFX6-NEXT: s_sub_i32 s10, s3, 24
+; GFX6-NEXT: s_cmp_lg_u32 s9, 0
+; GFX6-NEXT: s_cselect_b32 s3, s10, s3
+; GFX6-NEXT: s_cmp_ge_u32 s3, 24
+; GFX6-NEXT: s_cselect_b32 s9, 1, 0
+; GFX6-NEXT: s_sub_i32 s10, s3, 24
+; GFX6-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NEXT: s_cmp_lg_u32 s9, 0
+; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0
+; GFX6-NEXT: s_cselect_b32 s3, s10, s3
+; GFX6-NEXT: s_lshl_b32 s6, s6, 17
+; GFX6-NEXT: s_lshl_b32 s8, s8, 1
+; GFX6-NEXT: s_sub_i32 s9, 23, s3
+; GFX6-NEXT: s_or_b32 s6, s6, s8
+; GFX6-NEXT: s_lshl_b32 s4, s6, s9
+; GFX6-NEXT: s_lshr_b32 s3, s7, s3
+; GFX6-NEXT: s_or_b32 s4, s4, s3
+; GFX6-NEXT: v_readfirstlane_b32 s3, v0
+; GFX6-NEXT: s_mulk_i32 s3, 0xffe8
+; GFX6-NEXT: s_add_i32 s5, s5, s3
+; GFX6-NEXT: s_cmp_ge_u32 s5, 24
+; GFX6-NEXT: s_cselect_b32 s3, 1, 0
+; GFX6-NEXT: s_sub_i32 s6, s5, 24
+; GFX6-NEXT: s_cmp_lg_u32 s3, 0
+; GFX6-NEXT: s_cselect_b32 s3, s6, s5
+; GFX6-NEXT: s_cmp_ge_u32 s3, 24
+; GFX6-NEXT: s_cselect_b32 s5, 1, 0
+; GFX6-NEXT: s_sub_i32 s6, s3, 24
+; GFX6-NEXT: s_cmp_lg_u32 s5, 0
+; GFX6-NEXT: s_cselect_b32 s3, s6, s3
+; GFX6-NEXT: s_lshl_b32 s1, s1, 17
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: s_sub_i32 s5, 23, s3
+; GFX6-NEXT: s_or_b32 s0, s1, s0
+; GFX6-NEXT: s_lshl_b32 s0, s0, s5
+; GFX6-NEXT: s_lshr_b32 s1, s2, s3
+; GFX6-NEXT: s_bfe_u32 s2, s4, 0x80008
+; GFX6-NEXT: s_or_b32 s1, s0, s1
+; GFX6-NEXT: s_and_b32 s0, s4, 0xff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 8
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_bfe_u32 s2, s4, 0x80010
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s1, 0xff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_bfe_u32 s2, s1, 0x80008
+; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX6-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-NEXT: s_or_b32 s1, s2, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_v2i24:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_getpc_b64 s[8:9]
-; GFX8-NEXT: s_mov_b32 s8, s0
-; GFX8-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_u32 s8, s8, s6
-; GFX8-NEXT: s_addc_u32 s9, s9, 0
-; GFX8-NEXT: buffer_store_dword v0, off, s[8:11], 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:16
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:4
-; GFX8-NEXT: v_mov_b32_e32 v0, s5
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:20
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:12
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: buffer_store_dword v0, off, s[8:11], 0 offset:8
-; GFX8-NEXT: buffer_load_ubyte v0, off, s[8:11], 0 offset:2
-; GFX8-NEXT: buffer_load_ubyte v1, off, s[8:11], 0 offset:18
-; GFX8-NEXT: buffer_load_ubyte v2, off, s[8:11], 0 offset:4
-; GFX8-NEXT: buffer_load_ubyte v3, off, s[8:11], 0 offset:5
-; GFX8-NEXT: buffer_load_ubyte v4, off, s[8:11], 0 offset:21
-; GFX8-NEXT: buffer_load_ubyte v5, off, s[8:11], 0 offset:13
-; GFX8-NEXT: buffer_load_ubyte v6, off, s[8:11], 0 offset:12
-; GFX8-NEXT: buffer_load_ubyte v7, off, s[8:11], 0 offset:20
-; GFX8-NEXT: buffer_load_ubyte v8, off, s[8:11], 0 offset:19
-; GFX8-NEXT: buffer_load_ubyte v9, off, s[8:11], 0 offset:11
-; GFX8-NEXT: buffer_load_ubyte v10, off, s[8:11], 0 offset:10
-; GFX8-NEXT: buffer_load_ubyte v11, off, s[8:11], 0 offset:3
-; GFX8-NEXT: s_and_b32 s3, s4, 0xffff
-; GFX8-NEXT: s_mov_b32 s1, 0xaaaaaab
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_lshl_b32 s2, s2, 8
-; GFX8-NEXT: s_waitcnt vmcnt(11)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: s_waitcnt vmcnt(10)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: s_waitcnt vmcnt(9)
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: s_waitcnt vmcnt(8)
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: s_waitcnt vmcnt(7)
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_or_b32_e32 v1, s3, v1
-; GFX8-NEXT: v_mul_hi_u32 v1, v1, s1
-; GFX8-NEXT: s_waitcnt vmcnt(4)
-; GFX8-NEXT: v_lshlrev_b32_e32 v7, 8, v7
-; GFX8-NEXT: s_waitcnt vmcnt(3)
-; GFX8-NEXT: v_or_b32_e32 v7, v7, v8
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 8, v6
-; GFX8-NEXT: s_waitcnt vmcnt(2)
-; GFX8-NEXT: v_or_b32_e32 v6, v6, v9
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v11
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX8-NEXT: v_or_b32_e32 v3, v7, v4
-; GFX8-NEXT: v_mul_hi_u32 v3, v3, s1
-; GFX8-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 24, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v10, 24, v10
-; GFX8-NEXT: v_mul_u32_u24_e32 v3, 24, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 8, v6
-; GFX8-NEXT: v_sub_u32_e32 v1, vcc, s4, v1
-; GFX8-NEXT: v_sub_u32_e32 v3, vcc, v8, v3
-; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
-; GFX8-NEXT: v_or_b32_e32 v9, s2, v10
-; GFX8-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX8-NEXT: v_add_u32_e32 v1, vcc, 8, v1
-; GFX8-NEXT: v_add_u32_e32 v3, vcc, 8, v3
-; GFX8-NEXT: v_alignbit_b32 v0, v0, v9, v1
-; GFX8-NEXT: v_alignbit_b32 v1, v2, v4, v3
-; GFX8-NEXT: buffer_store_short v0, off, s[8:11], 0 offset:24
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 8, v1
-; GFX8-NEXT: buffer_store_byte v1, off, s[8:11], 0 offset:27
-; GFX8-NEXT: buffer_store_byte v0, off, s[8:11], 0 offset:26
-; GFX8-NEXT: buffer_store_byte v2, off, s[8:11], 0 offset:29
-; GFX8-NEXT: buffer_store_byte v3, off, s[8:11], 0 offset:28
-; GFX8-NEXT: buffer_load_dword v0, off, s[8:11], 0 offset:24
-; GFX8-NEXT: buffer_load_ushort v1, off, s[8:11], 0 offset:28
-; GFX8-NEXT: s_waitcnt vmcnt(1)
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-NEXT: s_lshr_b32 s9, s1, 8
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_lshr_b32 s6, s0, 8
+; GFX8-NEXT: s_lshr_b32 s8, s0, 24
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_and_b32 s6, s6, 0xff
+; GFX8-NEXT: s_or_b32 s1, s8, s1
+; GFX8-NEXT: s_lshr_b32 s8, s2, 8
+; GFX8-NEXT: s_lshr_b32 s7, s0, 16
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s6, s6, 8
+; GFX8-NEXT: s_and_b32 s8, s8, 0xff
+; GFX8-NEXT: s_or_b32 s0, s0, s6
+; GFX8-NEXT: s_and_b32 s6, s7, 0xff
+; GFX8-NEXT: s_and_b32 s7, s9, 0xff
+; GFX8-NEXT: s_lshr_b32 s9, s2, 16
+; GFX8-NEXT: s_lshr_b32 s10, s2, 24
+; GFX8-NEXT: s_and_b32 s2, s2, 0xff
+; GFX8-NEXT: s_lshl_b32 s8, s8, 8
+; GFX8-NEXT: s_or_b32 s2, s2, s8
+; GFX8-NEXT: s_and_b32 s8, s9, 0xff
+; GFX8-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX8-NEXT: s_lshr_b32 s11, s3, 8
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s8, s8, 16
+; GFX8-NEXT: s_and_b32 s3, s3, 0xff
+; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX8-NEXT: s_or_b32 s2, s2, s8
+; GFX8-NEXT: s_lshl_b32 s3, s3, 8
+; GFX8-NEXT: s_and_b32 s8, s11, 0xff
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_or_b32 s3, s10, s3
+; GFX8-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s8, s8, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s8
+; GFX8-NEXT: s_lshr_b32 s8, s4, 8
+; GFX8-NEXT: s_and_b32 s8, s8, 0xff
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: s_lshr_b32 s9, s4, 16
+; GFX8-NEXT: s_lshr_b32 s10, s4, 24
+; GFX8-NEXT: s_and_b32 s4, s4, 0xff
+; GFX8-NEXT: s_lshl_b32 s8, s8, 8
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: s_or_b32 s4, s4, s8
+; GFX8-NEXT: s_and_b32 s8, s9, 0xff
+; GFX8-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_lshl_b32 s8, s8, 16
+; GFX8-NEXT: s_or_b32 s4, s4, s8
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: s_mul_i32 s9, s8, 0xffffffe8
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s9
+; GFX8-NEXT: s_lshr_b32 s11, s5, 8
+; GFX8-NEXT: s_and_b32 s5, s5, 0xff
+; GFX8-NEXT: s_lshl_b32 s5, s5, 8
+; GFX8-NEXT: s_or_b32 s5, s10, s5
+; GFX8-NEXT: v_readfirstlane_b32 s10, v0
+; GFX8-NEXT: s_add_i32 s8, s8, s10
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT: s_and_b32 s9, s11, 0xff
+; GFX8-NEXT: s_and_b32 s9, 0xffff, s9
+; GFX8-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX8-NEXT: s_lshl_b32 s9, s9, 16
+; GFX8-NEXT: s_or_b32 s5, s5, s9
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
+; GFX8-NEXT: s_mulk_i32 s9, 0xffe8
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX8-NEXT: s_add_i32 s4, s4, s9
+; GFX8-NEXT: s_cmp_ge_u32 s4, 24
+; GFX8-NEXT: s_cselect_b32 s9, 1, 0
+; GFX8-NEXT: s_sub_i32 s10, s4, 24
+; GFX8-NEXT: s_cmp_lg_u32 s9, 0
+; GFX8-NEXT: s_cselect_b32 s4, s10, s4
+; GFX8-NEXT: s_cmp_ge_u32 s4, 24
+; GFX8-NEXT: s_cselect_b32 s9, 1, 0
+; GFX8-NEXT: s_sub_i32 s10, s4, 24
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: s_cmp_lg_u32 s9, 0
+; GFX8-NEXT: v_mul_hi_u32 v0, s5, v0
+; GFX8-NEXT: s_cselect_b32 s4, s10, s4
+; GFX8-NEXT: s_lshl_b32 s6, s6, 17
+; GFX8-NEXT: s_lshl_b32 s0, s0, 1
+; GFX8-NEXT: s_sub_i32 s9, 23, s4
+; GFX8-NEXT: s_or_b32 s0, s6, s0
+; GFX8-NEXT: s_lshl_b32 s0, s0, s9
+; GFX8-NEXT: s_lshr_b32 s2, s2, s4
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: s_mulk_i32 s2, 0xffe8
+; GFX8-NEXT: s_add_i32 s5, s5, s2
+; GFX8-NEXT: s_cmp_ge_u32 s5, 24
+; GFX8-NEXT: s_cselect_b32 s2, 1, 0
+; GFX8-NEXT: s_sub_i32 s4, s5, 24
+; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_cselect_b32 s2, s4, s5
+; GFX8-NEXT: s_cmp_ge_u32 s2, 24
+; GFX8-NEXT: s_cselect_b32 s4, 1, 0
+; GFX8-NEXT: s_sub_i32 s5, s2, 24
+; GFX8-NEXT: s_cmp_lg_u32 s4, 0
+; GFX8-NEXT: s_cselect_b32 s2, s5, s2
+; GFX8-NEXT: s_lshl_b32 s5, s7, 17
+; GFX8-NEXT: s_lshl_b32 s1, s1, 1
+; GFX8-NEXT: s_sub_i32 s4, 23, s2
+; GFX8-NEXT: s_or_b32 s1, s5, s1
+; GFX8-NEXT: s_lshl_b32 s1, s1, s4
+; GFX8-NEXT: s_lshr_b32 s2, s3, s2
+; GFX8-NEXT: s_bfe_u32 s3, s0, 0x80008
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s3, s3, 8
+; GFX8-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_or_b32 s0, s2, s0
+; GFX8-NEXT: s_and_b32 s2, s1, 0xff
+; GFX8-NEXT: s_lshl_b32 s2, s2, 24
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_bfe_u32 s2, s1, 0x80008
+; GFX8-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_or_b32 s1, s2, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v2i24:
; GFX11: ; %bb.0:
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s5
-; GFX11-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX11-NEXT: scratch_store_b32 off, v0, off
-; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, 24
+; GFX11-NEXT: s_lshr_b32 s9, s1, 8
+; GFX11-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-NEXT: s_lshr_b32 s6, s0, 8
+; GFX11-NEXT: s_lshr_b32 s8, s0, 24
+; GFX11-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX11-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-NEXT: s_clause 0x4
-; GFX11-NEXT: scratch_store_b32 off, v0, off offset:16
-; GFX11-NEXT: scratch_store_b16 off, v1, off offset:4
-; GFX11-NEXT: scratch_store_b16 off, v2, off offset:20
-; GFX11-NEXT: scratch_store_b32 off, v3, off offset:8
-; GFX11-NEXT: scratch_store_b16 off, v4, off offset:12
-; GFX11-NEXT: s_clause 0xb
-; GFX11-NEXT: scratch_load_u8 v0, off, off offset:21
-; GFX11-NEXT: scratch_load_u8 v1, off, off offset:20
-; GFX11-NEXT: scratch_load_u8 v2, off, off offset:19
-; GFX11-NEXT: scratch_load_u8 v3, off, off offset:18
-; GFX11-NEXT: scratch_load_u8 v4, off, off offset:11
-; GFX11-NEXT: scratch_load_u8 v5, off, off offset:12
-; GFX11-NEXT: scratch_load_u8 v6, off, off offset:5
-; GFX11-NEXT: scratch_load_u8 v7, off, off offset:4
-; GFX11-NEXT: scratch_load_u8 v8, off, off offset:3
-; GFX11-NEXT: scratch_load_u8 v9, off, off offset:13
-; GFX11-NEXT: scratch_load_u8 v10, off, off offset:2
-; GFX11-NEXT: scratch_load_u8 v11, off, off offset:10
-; GFX11-NEXT: s_waitcnt vmcnt(11)
-; GFX11-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-NEXT: s_waitcnt vmcnt(10)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX11-NEXT: s_waitcnt vmcnt(9)
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_or3_b32 v0, v1, v2, v0
-; GFX11-NEXT: s_waitcnt vmcnt(8)
-; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX11-NEXT: s_waitcnt vmcnt(6)
-; GFX11-NEXT: v_lshl_or_b32 v3, v5, 8, v4
-; GFX11-NEXT: s_waitcnt vmcnt(5)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v6
-; GFX11-NEXT: s_waitcnt vmcnt(4)
-; GFX11-NEXT: v_lshlrev_b32_e32 v5, 8, v7
-; GFX11-NEXT: v_mul_hi_u32 v0, 0xaaaaaab, v0
-; GFX11-NEXT: v_and_or_b32 v1, 0xffff, s4, v1
-; GFX11-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mul_hi_u32 v1, 0xaaaaaab, v1
-; GFX11-NEXT: s_waitcnt vmcnt(2)
-; GFX11-NEXT: v_lshl_or_b32 v3, v9, 24, v3
-; GFX11-NEXT: v_mul_u32_u24_e32 v0, 24, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-NEXT: v_sub_nc_u32_e32 v0, v2, v0
-; GFX11-NEXT: v_mul_u32_u24_e32 v1, 24, v1
-; GFX11-NEXT: v_or3_b32 v2, v5, v8, v4
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_lshlrev_b32_e32 v4, 16, v10
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_lshl_or_b32 v5, v11, 24, s1
-; GFX11-NEXT: v_add_nc_u32_e32 v0, 8, v0
-; GFX11-NEXT: v_sub_nc_u32_e32 v1, s4, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_alignbit_b32 v0, v2, v3, v0.l
-; GFX11-NEXT: v_and_or_b32 v2, 0xffff, s0, v4
-; GFX11-NEXT: v_add_nc_u32_e32 v1, 8, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-NEXT: v_alignbit_b32 v1, v2, v5, v1.l
-; GFX11-NEXT: s_clause 0x4
-; GFX11-NEXT: scratch_store_d16_hi_b8 off, v0, off offset:29
-; GFX11-NEXT: scratch_store_b8 off, v3, off offset:28
-; GFX11-NEXT: scratch_store_b8 off, v0, off offset:27
-; GFX11-NEXT: scratch_store_d16_hi_b8 off, v1, off offset:26
-; GFX11-NEXT: scratch_store_b16 off, v1, off offset:24
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: scratch_load_b32 v0, off, off offset:24
-; GFX11-NEXT: scratch_load_u16 v1, off, off offset:28
-; GFX11-NEXT: s_waitcnt vmcnt(1)
-; GFX11-NEXT: v_readfirstlane_b32 s0, v0
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-NEXT: s_lshr_b32 s11, s3, 8
+; GFX11-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-NEXT: s_and_b32 s6, s6, 0xff
+; GFX11-NEXT: s_or_b32 s1, s8, s1
+; GFX11-NEXT: s_lshr_b32 s8, s2, 8
+; GFX11-NEXT: s_lshr_b32 s10, s2, 24
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-NEXT: s_lshr_b32 s7, s0, 16
+; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
+; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-NEXT: s_lshl_b32 s6, s6, 8
+; GFX11-NEXT: s_and_b32 s8, s8, 0xff
+; GFX11-NEXT: s_or_b32 s3, s10, s3
+; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX11-NEXT: s_lshr_b32 s10, s4, 8
+; GFX11-NEXT: s_or_b32 s0, s0, s6
+; GFX11-NEXT: s_and_b32 s6, s7, 0xff
+; GFX11-NEXT: s_and_b32 s7, s9, 0xff
+; GFX11-NEXT: v_readfirstlane_b32 s13, v0
+; GFX11-NEXT: s_lshr_b32 s9, s2, 16
+; GFX11-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-NEXT: s_lshl_b32 s8, s8, 8
+; GFX11-NEXT: s_and_b32 s10, s10, 0xff
+; GFX11-NEXT: s_or_b32 s2, s2, s8
+; GFX11-NEXT: s_and_b32 s8, s9, 0xff
+; GFX11-NEXT: s_and_b32 s9, s11, 0xff
+; GFX11-NEXT: s_lshr_b32 s11, s4, 16
+; GFX11-NEXT: s_lshr_b32 s12, s4, 24
+; GFX11-NEXT: s_and_b32 s4, s4, 0xff
+; GFX11-NEXT: s_lshl_b32 s10, s10, 8
+; GFX11-NEXT: s_and_b32 s11, s11, 0xff
+; GFX11-NEXT: s_or_b32 s4, s4, s10
+; GFX11-NEXT: s_mul_i32 s10, s13, 0xffffffe8
+; GFX11-NEXT: s_pack_ll_b32_b16 s4, s4, s11
+; GFX11-NEXT: s_mul_hi_u32 s10, s13, s10
+; GFX11-NEXT: s_lshr_b32 s14, s5, 8
+; GFX11-NEXT: s_add_i32 s13, s13, s10
+; GFX11-NEXT: s_and_b32 s5, s5, 0xff
+; GFX11-NEXT: s_mul_hi_u32 s10, s4, s13
+; GFX11-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-NEXT: s_mulk_i32 s10, 0xffe8
+; GFX11-NEXT: s_or_b32 s5, s12, s5
+; GFX11-NEXT: s_add_i32 s4, s4, s10
+; GFX11-NEXT: s_and_b32 s10, s14, 0xff
+; GFX11-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-NEXT: s_pack_ll_b32_b16 s0, s0, s6
+; GFX11-NEXT: s_cselect_b32 s11, 1, 0
+; GFX11-NEXT: s_sub_i32 s12, s4, 24
+; GFX11-NEXT: s_cmp_lg_u32 s11, 0
+; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, s8
+; GFX11-NEXT: s_cselect_b32 s4, s12, s4
+; GFX11-NEXT: s_pack_ll_b32_b16 s5, s5, s10
+; GFX11-NEXT: s_cmp_ge_u32 s4, 24
+; GFX11-NEXT: s_pack_ll_b32_b16 s1, s1, s7
+; GFX11-NEXT: s_cselect_b32 s6, 1, 0
+; GFX11-NEXT: s_sub_i32 s8, s4, 24
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-NEXT: s_pack_ll_b32_b16 s3, s3, s9
+; GFX11-NEXT: s_cselect_b32 s4, s8, s4
+; GFX11-NEXT: s_mul_hi_u32 s8, s5, s13
+; GFX11-NEXT: s_sub_i32 s6, 23, s4
+; GFX11-NEXT: s_lshl_b32 s0, s0, 1
+; GFX11-NEXT: s_mulk_i32 s8, 0xffe8
+; GFX11-NEXT: s_lshl_b32 s0, s0, s6
+; GFX11-NEXT: s_lshr_b32 s2, s2, s4
+; GFX11-NEXT: s_add_i32 s5, s5, s8
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_cmp_ge_u32 s5, 24
+; GFX11-NEXT: s_cselect_b32 s2, 1, 0
+; GFX11-NEXT: s_sub_i32 s4, s5, 24
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-NEXT: s_cselect_b32 s2, s4, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_cmp_ge_u32 s2, 24
+; GFX11-NEXT: s_cselect_b32 s4, 1, 0
+; GFX11-NEXT: s_sub_i32 s5, s2, 24
+; GFX11-NEXT: s_cmp_lg_u32 s4, 0
+; GFX11-NEXT: s_cselect_b32 s2, s5, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, 1
+; GFX11-NEXT: s_sub_i32 s4, 23, s2
+; GFX11-NEXT: s_lshr_b32 s2, s3, s2
+; GFX11-NEXT: s_lshl_b32 s1, s1, s4
+; GFX11-NEXT: s_and_b32 s3, s0, 0xff
+; GFX11-NEXT: s_or_b32 s1, s1, s2
+; GFX11-NEXT: s_bfe_u32 s2, s0, 0x80008
+; GFX11-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX11-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-NEXT: s_lshl_b32 s0, s0, 16
+; GFX11-NEXT: s_or_b32 s2, s3, s2
+; GFX11-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-NEXT: s_or_b32 s0, s2, s0
+; GFX11-NEXT: s_lshl_b32 s2, s3, 24
+; GFX11-NEXT: s_bfe_u32 s3, s1, 0x80010
+; GFX11-NEXT: s_bfe_u32 s1, s1, 0x80008
+; GFX11-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-NEXT: s_or_b32 s0, s0, s2
+; GFX11-NEXT: s_or_b32 s1, s1, s3
; GFX11-NEXT: ; return to shader part epilog
; GFX9-LABEL: s_fshr_v2i24:
; GFX9: ; %bb.0:
@@ -3552,43 +3710,50 @@ define amdgpu_ps half @v_fshr_i16_vss(i16 %lhs, i16 inreg %rhs, i16 inreg %amt)
define amdgpu_ps i32 @s_fshr_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
; GFX6-LABEL: s_fshr_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_mov_b32 s5, s0
-; GFX6-NEXT: s_lshr_b32 s0, s2, 16
-; GFX6-NEXT: s_or_b32 s2, s2, 16
-; GFX6-NEXT: s_lshl_b32 s4, s1, 16
-; GFX6-NEXT: s_and_b32 s2, s2, 31
-; GFX6-NEXT: s_or_b32 s0, s0, 16
-; GFX6-NEXT: s_lshr_b32 s3, s5, 16
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s2
-; GFX6-NEXT: s_and_b32 s2, s1, 0xffff0000
-; GFX6-NEXT: s_and_b32 s0, s0, 31
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s0, s4, s0
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_lshr_b32 s4, s1, 16
+; GFX6-NEXT: s_lshr_b32 s5, s2, 16
+; GFX6-NEXT: s_and_b32 s6, s2, 15
+; GFX6-NEXT: s_andn2_b32 s2, 15, s2
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, s2
+; GFX6-NEXT: s_lshr_b32 s1, s1, s6
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s5, 15
+; GFX6-NEXT: s_andn2_b32 s2, 15, s5
+; GFX6-NEXT: s_lshl_b32 s3, s3, 1
+; GFX6-NEXT: s_lshl_b32 s2, s3, s2
+; GFX6-NEXT: s_lshr_b32 s1, s4, s1
+; GFX6-NEXT: s_or_b32 s1, s2, s1
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s3, s2, 15
-; GFX8-NEXT: s_and_b32 s4, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s3, s4, s3
-; GFX8-NEXT: s_andn2_b32 s4, 15, s2
-; GFX8-NEXT: s_lshl_b32 s5, s0, 1
-; GFX8-NEXT: s_lshl_b32 s4, s5, s4
-; GFX8-NEXT: s_or_b32 s3, s4, s3
-; GFX8-NEXT: s_lshr_b32 s4, s2, 16
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX8-NEXT: s_andn2_b32 s4, 15, s4
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: s_lshr_b32 s4, s1, 16
+; GFX8-NEXT: s_lshr_b32 s5, s2, 16
+; GFX8-NEXT: s_and_b32 s6, s2, 15
+; GFX8-NEXT: s_andn2_b32 s2, 15, s2
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40010
-; GFX8-NEXT: s_lshl_b32 s0, s0, s4
-; GFX8-NEXT: s_lshr_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s2
+; GFX8-NEXT: s_lshr_b32 s1, s1, s6
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s5, 15
+; GFX8-NEXT: s_andn2_b32 s2, 15, s5
+; GFX8-NEXT: s_lshl_b32 s3, s3, 1
+; GFX8-NEXT: s_lshl_b32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s1, s4, s1
+; GFX8-NEXT: s_or_b32 s2, s2, s1
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s3, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v2i16:
@@ -3664,35 +3829,53 @@ define <2 x i16> @v_fshr_v2i16(<2 x i16> %lhs, <2 x i16> %rhs, <2 x i16> %amt) {
; GFX6-LABEL: v_fshr_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v6, 15, v2
+; GFX6-NEXT: v_xor_b32_e32 v2, -1, v2
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff0000, v1
-; GFX6-NEXT: v_or_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v3, v3, v5, v4
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v6, v1
+; GFX6-NEXT: v_xor_b32_e32 v2, -1, v5
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 15, v5
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 1, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, 1
-; GFX8-NEXT: v_mov_b32_e32 v5, -1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v4, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_xor_b32_sdwa v5, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_sdwa v3, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, v5, v4
-; GFX8-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, 1, v0
; GFX8-NEXT: v_xor_b32_e32 v4, -1, v2
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v4, v0
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, v2, v1
+; GFX8-NEXT: v_and_b32_e32 v3, 15, v2
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v5, 1, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, v4, v5
+; GFX8-NEXT: v_lshrrev_b16_e32 v3, v3, v1
+; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX8-NEXT: v_mov_b32_e32 v4, 15
+; GFX8-NEXT: v_mov_b32_e32 v5, -1
+; GFX8-NEXT: v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v5, 1
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, v2, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshr_v2i16:
@@ -3739,24 +3922,28 @@ define <2 x i16> @v_fshr_v2i16_4_8(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_alignbit_b32 v2, v2, v1, 24
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v1, 20
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 12, v0
+; GFX6-NEXT: v_bfe_u32 v3, v1, 4, 12
+; GFX6-NEXT: v_alignbit_b32 v1, v2, v1, 24
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v2i16_4_8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s4, 0xc0c0207
-; GFX8-NEXT: v_perm_b32 v2, v1, v0, s4
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, 4, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v2
+; GFX8-NEXT: v_lshrrev_b16_e32 v3, 4, v1
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
; GFX8-NEXT: v_lshlrev_b16_e32 v0, 12, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshr_v2i16_4_8:
@@ -3791,37 +3978,52 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
; GFX6-LABEL: v_fshr_v2i16_ssv:
; GFX6: ; %bb.0:
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: s_and_b32 s3, s1, 0xffff0000
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v0
+; GFX6-NEXT: v_xor_b32_e32 v0, -1, v0
; GFX6-NEXT: s_lshr_b32 s2, s0, 16
-; GFX6-NEXT: v_or_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_mov_b32_e32 v2, s3
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: v_alignbit_b32 v1, s2, v2, v1
-; GFX6-NEXT: v_or_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
+; GFX6-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX6-NEXT: v_lshr_b32_e32 v2, s0, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v1
+; GFX6-NEXT: v_xor_b32_e32 v1, -1, v1
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: v_and_b32_e32 v1, 15, v1
+; GFX6-NEXT: s_lshl_b32 s0, s2, 1
+; GFX6-NEXT: v_lshl_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_lshr_b32_e32 v2, s3, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_v2i16_ssv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, -1
-; GFX8-NEXT: s_bfe_u32 s2, s0, 0x100010
-; GFX8-NEXT: v_xor_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: s_lshl_b32 s2, s2, 1
-; GFX8-NEXT: v_lshrrev_b16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e64 v2, v2, s2
-; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_xor_b32_e32 v2, -1, v0
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: v_and_b32_e32 v1, 15, v0
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v2
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
; GFX8-NEXT: v_lshlrev_b16_e64 v2, v2, s0
-; GFX8-NEXT: v_lshrrev_b16_e64 v0, v0, s1
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b16_e64 v1, v1, s1
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 15
+; GFX8-NEXT: v_mov_b32_e32 v3, -1
+; GFX8-NEXT: v_and_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: v_and_b32_e32 v0, 15, v0
+; GFX8-NEXT: s_lshl_b32 s0, s2, 1
+; GFX8-NEXT: v_lshlrev_b16_e64 v0, v0, s0
+; GFX8-NEXT: v_lshrrev_b16_e64 v2, v2, s3
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: v_fshr_v2i16_ssv:
@@ -3872,39 +4074,47 @@ define amdgpu_ps float @v_fshr_v2i16_ssv(<2 x i16> inreg %lhs, <2 x i16> inreg %
define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <2 x i16> inreg %amt) {
; GFX6-LABEL: v_fshr_v2i16_svs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s3, s1, 16
-; GFX6-NEXT: s_or_b32 s3, s3, 16
; GFX6-NEXT: s_lshr_b32 s2, s0, 16
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff0000, v0
-; GFX6-NEXT: v_mov_b32_e32 v2, s3
-; GFX6-NEXT: s_or_b32 s1, s1, 16
-; GFX6-NEXT: v_alignbit_b32 v1, s2, v1, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_alignbit_b32 v0, s0, v0, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: s_and_b32 s4, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: s_lshl_b32 s0, s0, s1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s4, v0
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: s_and_b32 s0, s3, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s3
+; GFX6-NEXT: s_lshl_b32 s2, s2, 1
+; GFX6-NEXT: s_lshl_b32 s1, s2, s1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_or_b32_e32 v1, s1, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_v2i16_svs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
-; GFX8-NEXT: s_lshl_b32 s3, s0, 1
-; GFX8-NEXT: s_lshl_b32 s2, s3, s2
-; GFX8-NEXT: s_and_b32 s3, s1, 15
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, s3, v0
-; GFX8-NEXT: v_or_b32_e32 v1, s2, v1
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX8-NEXT: s_andn2_b32 s2, 15, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_and_b32 s4, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x40010
-; GFX8-NEXT: s_lshl_b32 s0, s0, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: s_lshl_b32 s0, s0, s1
+; GFX8-NEXT: v_lshrrev_b16_e32 v1, s4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT: s_and_b32 s0, s3, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s3
+; GFX8-NEXT: s_lshl_b32 s2, s2, 1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: s_lshl_b32 s1, s2, s1
+; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, s1, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
@@ -3966,40 +4176,48 @@ define amdgpu_ps float @v_fshr_v2i16_svs(<2 x i16> inreg %lhs, <2 x i16> %rhs, <
define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <2 x i16> inreg %amt) {
; GFX6-LABEL: v_fshr_v2i16_vss:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s2, s1, 16
-; GFX6-NEXT: s_or_b32 s2, s2, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: s_and_b32 s3, s0, 0xffff0000
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
-; GFX6-NEXT: s_or_b32 s1, s1, 16
-; GFX6-NEXT: v_alignbit_b32 v1, v1, s3, v2
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
-; GFX6-NEXT: v_alignbit_b32 v0, v0, s0, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: s_and_b32 s4, s1, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, s1, v0
+; GFX6-NEXT: s_lshr_b32 s0, s0, s4
+; GFX6-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX6-NEXT: s_and_b32 s0, s3, 15
+; GFX6-NEXT: s_andn2_b32 s1, 15, s3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 1, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, s1, v1
+; GFX6-NEXT: s_lshr_b32 s0, s2, s0
+; GFX6-NEXT: v_or_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: v_fshr_v2i16_vss:
; GFX8: ; %bb.0:
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_and_b32 s4, s1, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s1
; GFX8-NEXT: v_lshlrev_b16_e32 v1, 1, v0
-; GFX8-NEXT: s_andn2_b32 s2, 15, s1
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, s2, v1
-; GFX8-NEXT: s_and_b32 s2, s1, 15
-; GFX8-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX8-NEXT: s_lshr_b32 s2, s3, s2
-; GFX8-NEXT: v_or_b32_e32 v1, s2, v1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, s1, v1
+; GFX8-NEXT: s_lshr_b32 s0, s0, s4
; GFX8-NEXT: v_mov_b32_e32 v2, 1
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x40010
+; GFX8-NEXT: v_or_b32_e32 v1, s0, v1
+; GFX8-NEXT: s_and_b32 s0, s3, 15
+; GFX8-NEXT: s_andn2_b32 s1, 15, s3
; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_andn2_b32 s2, 15, s2
-; GFX8-NEXT: s_lshr_b32 s0, s0, s1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, s2, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, s1, v0
+; GFX8-NEXT: s_lshr_b32 s0, s2, s0
+; GFX8-NEXT: v_or_b32_e32 v0, s0, v0
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: ; return to shader part epilog
;
@@ -4055,75 +4273,66 @@ define amdgpu_ps float @v_fshr_v2i16_vss(<2 x i16> %lhs, <2 x i16> inreg %rhs, <
define amdgpu_ps i48 @s_fshr_v3i16(<3 x i16> inreg %lhs, <3 x i16> inreg %rhs, <3 x i16> inreg %amt) {
; GFX6-LABEL: s_fshr_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s6, s5, 16
-; GFX6-NEXT: s_or_b32 s5, s5, 16
-; GFX6-NEXT: s_mov_b32 s7, s0
-; GFX6-NEXT: s_lshr_b32 s9, s0, 16
-; GFX6-NEXT: s_lshl_b32 s0, s3, 16
-; GFX6-NEXT: s_and_b32 s5, s5, 31
-; GFX6-NEXT: s_lshr_b32 s11, s1, 16
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s5
-; GFX6-NEXT: s_and_b32 s5, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s6, 16
-; GFX6-NEXT: s_and_b32 s10, s3, 0xffff0000
-; GFX6-NEXT: s_and_b32 s0, s0, 31
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[10:11], s0
-; GFX6-NEXT: s_lshr_b32 s12, s4, 16
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s1, s5, s0
-; GFX6-NEXT: s_lshl_b32 s6, s2, 16
-; GFX6-NEXT: s_or_b32 s0, s4, 16
-; GFX6-NEXT: s_and_b32 s8, s2, 0xffff0000
-; GFX6-NEXT: s_or_b32 s2, s12, 16
-; GFX6-NEXT: s_and_b32 s0, s0, 31
-; GFX6-NEXT: s_and_b32 s2, s2, 31
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], s0
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[8:9], s2
-; GFX6-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshr_b32 s7, s2, 16
+; GFX6-NEXT: s_lshr_b32 s8, s4, 16
+; GFX6-NEXT: s_and_b32 s9, s4, 15
+; GFX6-NEXT: s_andn2_b32 s4, 15, s4
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, s4
+; GFX6-NEXT: s_lshr_b32 s2, s2, s9
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s8, 15
+; GFX6-NEXT: s_andn2_b32 s4, 15, s8
+; GFX6-NEXT: s_lshl_b32 s6, s6, 1
+; GFX6-NEXT: s_lshl_b32 s4, s6, s4
+; GFX6-NEXT: s_lshr_b32 s2, s7, s2
+; GFX6-NEXT: s_or_b32 s2, s4, s2
+; GFX6-NEXT: s_and_b32 s4, s5, 15
+; GFX6-NEXT: s_andn2_b32 s5, 15, s5
+; GFX6-NEXT: s_lshl_b32 s1, s1, 1
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, s5
+; GFX6-NEXT: s_lshr_b32 s3, s3, s4
+; GFX6-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX6-NEXT: s_or_b32 s1, s1, s3
+; GFX6-NEXT: s_and_b32 s0, 0xffff, s0
; GFX6-NEXT: s_lshl_b32 s2, s2, 16
; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_v3i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s6, s5, 15
-; GFX8-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX8-NEXT: s_lshr_b32 s6, s7, s6
-; GFX8-NEXT: s_andn2_b32 s7, 15, s5
-; GFX8-NEXT: s_lshl_b32 s8, s1, 1
-; GFX8-NEXT: s_lshl_b32 s7, s8, s7
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshr_b32 s7, s5, 16
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x100010
-; GFX8-NEXT: s_andn2_b32 s7, 15, s7
-; GFX8-NEXT: s_lshl_b32 s1, s1, 1
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX8-NEXT: s_lshl_b32 s1, s1, s7
-; GFX8-NEXT: s_lshr_b32 s3, s3, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s3
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_and_b32 s3, s4, 15
-; GFX8-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s6, s1
-; GFX8-NEXT: s_lshr_b32 s3, s5, s3
-; GFX8-NEXT: s_andn2_b32 s5, 15, s4
-; GFX8-NEXT: s_lshl_b32 s6, s0, 1
-; GFX8-NEXT: s_lshl_b32 s5, s6, s5
-; GFX8-NEXT: s_or_b32 s3, s5, s3
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX8-NEXT: s_andn2_b32 s5, 15, s5
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: s_lshr_b32 s7, s2, 16
+; GFX8-NEXT: s_lshr_b32 s8, s4, 16
+; GFX8-NEXT: s_and_b32 s9, s4, 15
+; GFX8-NEXT: s_andn2_b32 s4, 15, s4
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX8-NEXT: s_lshl_b32 s0, s0, s5
-; GFX8-NEXT: s_lshr_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s0, s0, s4
+; GFX8-NEXT: s_lshr_b32 s2, s2, s9
; GFX8-NEXT: s_or_b32 s0, s0, s2
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s3, s0
+; GFX8-NEXT: s_and_b32 s2, s8, 15
+; GFX8-NEXT: s_andn2_b32 s4, 15, s8
+; GFX8-NEXT: s_lshl_b32 s6, s6, 1
+; GFX8-NEXT: s_lshl_b32 s4, s6, s4
+; GFX8-NEXT: s_lshr_b32 s2, s7, s2
+; GFX8-NEXT: s_or_b32 s4, s4, s2
+; GFX8-NEXT: s_and_b32 s2, s5, 15
+; GFX8-NEXT: s_andn2_b32 s5, 15, s5
+; GFX8-NEXT: s_lshl_b32 s1, s1, 1
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s1, s1, s5
+; GFX8-NEXT: s_lshr_b32 s2, s3, s2
+; GFX8-NEXT: s_or_b32 s1, s1, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s4
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v3i16:
@@ -4307,44 +4516,70 @@ define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
; GFX6-LABEL: v_fshr_v3i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v9, 15, v4
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff0000, v2
-; GFX6-NEXT: v_or_b32_e32 v7, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v1, v1, v3, v5
-; GFX6-NEXT: v_or_b32_e32 v3, 16, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_alignbit_b32 v6, v6, v8, v7
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v2, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v9, v2
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v8
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v8
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 1, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v7
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v5
+; GFX6-NEXT: v_xor_b32_e32 v5, -1, v5
+; GFX6-NEXT: v_and_b32_e32 v5, 15, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 1, v1
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, v5, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v6
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v3i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, 1
+; GFX8-NEXT: v_xor_b32_e32 v7, -1, v4
+; GFX8-NEXT: v_and_b32_e32 v6, 15, v4
+; GFX8-NEXT: v_and_b32_e32 v7, 15, v7
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, 1, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v7, v7, v8
+; GFX8-NEXT: v_lshrrev_b16_e32 v6, v6, v2
+; GFX8-NEXT: v_or_b32_e32 v6, v7, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, 15
; GFX8-NEXT: v_mov_b32_e32 v8, -1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v7, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_xor_b32_sdwa v8, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_sdwa v6, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v7, v8, v7
-; GFX8-NEXT: v_or_b32_sdwa v6, v7, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, 1, v1
-; GFX8-NEXT: v_xor_b32_e32 v7, -1, v5
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, v7, v1
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, 1, v0
-; GFX8-NEXT: v_xor_b32_e32 v3, -1, v4
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v3, v0
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, v4, v2
+; GFX8-NEXT: v_and_b32_sdwa v7, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v4, v4, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v8, 1
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v8, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, v4, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_xor_b32_e32 v4, -1, v5
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v5
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, 1, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, v4, v1
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v2, v3
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshr_v3i16:
@@ -4411,75 +4646,90 @@ define <3 x half> @v_fshr_v3i16(<3 x i16> %lhs, <3 x i16> %rhs, <3 x i16> %amt)
define amdgpu_ps <2 x i32> @s_fshr_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs, <4 x i16> inreg %amt) {
; GFX6-LABEL: s_fshr_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s6, s5, 16
-; GFX6-NEXT: s_or_b32 s5, s5, 16
-; GFX6-NEXT: s_mov_b32 s7, s0
-; GFX6-NEXT: s_lshr_b32 s9, s0, 16
-; GFX6-NEXT: s_lshl_b32 s0, s3, 16
-; GFX6-NEXT: s_and_b32 s5, s5, 31
-; GFX6-NEXT: s_lshr_b32 s11, s1, 16
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[0:1], s5
-; GFX6-NEXT: s_and_b32 s5, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s6, 16
-; GFX6-NEXT: s_and_b32 s10, s3, 0xffff0000
-; GFX6-NEXT: s_and_b32 s0, s0, 31
-; GFX6-NEXT: s_lshr_b64 s[0:1], s[10:11], s0
-; GFX6-NEXT: s_lshr_b32 s12, s4, 16
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_or_b32 s1, s5, s0
-; GFX6-NEXT: s_lshl_b32 s6, s2, 16
-; GFX6-NEXT: s_or_b32 s0, s4, 16
-; GFX6-NEXT: s_and_b32 s8, s2, 0xffff0000
-; GFX6-NEXT: s_or_b32 s2, s12, 16
-; GFX6-NEXT: s_and_b32 s0, s0, 31
-; GFX6-NEXT: s_and_b32 s2, s2, 31
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], s0
-; GFX6-NEXT: s_lshr_b64 s[2:3], s[8:9], s2
-; GFX6-NEXT: s_and_b32 s0, s4, 0xffff
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshr_b32 s8, s2, 16
+; GFX6-NEXT: s_lshr_b32 s10, s4, 16
+; GFX6-NEXT: s_and_b32 s12, s4, 15
+; GFX6-NEXT: s_andn2_b32 s4, 15, s4
+; GFX6-NEXT: s_lshl_b32 s0, s0, 1
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, s4
+; GFX6-NEXT: s_lshr_b32 s2, s2, s12
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s10, 15
+; GFX6-NEXT: s_andn2_b32 s4, 15, s10
+; GFX6-NEXT: s_lshl_b32 s6, s6, 1
+; GFX6-NEXT: s_lshl_b32 s4, s6, s4
+; GFX6-NEXT: s_lshr_b32 s2, s8, s2
+; GFX6-NEXT: s_or_b32 s2, s4, s2
+; GFX6-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX6-NEXT: s_and_b32 s0, 0xffff, s0
; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_lshr_b32 s7, s1, 16
+; GFX6-NEXT: s_lshr_b32 s9, s3, 16
; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s5, 15
+; GFX6-NEXT: s_andn2_b32 s4, 15, s5
+; GFX6-NEXT: s_lshl_b32 s1, s1, 1
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_lshr_b32 s11, s5, 16
+; GFX6-NEXT: s_lshl_b32 s1, s1, s4
+; GFX6-NEXT: s_lshr_b32 s2, s3, s2
+; GFX6-NEXT: s_or_b32 s1, s1, s2
+; GFX6-NEXT: s_and_b32 s2, s11, 15
+; GFX6-NEXT: s_andn2_b32 s3, 15, s11
+; GFX6-NEXT: s_lshl_b32 s4, s7, 1
+; GFX6-NEXT: s_lshl_b32 s3, s4, s3
+; GFX6-NEXT: s_lshr_b32 s2, s9, s2
+; GFX6-NEXT: s_or_b32 s2, s3, s2
+; GFX6-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX6-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s2
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_fshr_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s6, s5, 15
-; GFX8-NEXT: s_and_b32 s7, s3, 0xffff
-; GFX8-NEXT: s_lshr_b32 s6, s7, s6
-; GFX8-NEXT: s_andn2_b32 s7, 15, s5
-; GFX8-NEXT: s_lshl_b32 s8, s1, 1
-; GFX8-NEXT: s_lshl_b32 s7, s8, s7
-; GFX8-NEXT: s_or_b32 s6, s7, s6
-; GFX8-NEXT: s_lshr_b32 s7, s5, 16
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x100010
-; GFX8-NEXT: s_andn2_b32 s7, 15, s7
-; GFX8-NEXT: s_lshl_b32 s1, s1, 1
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_bfe_u32 s5, s5, 0x40010
-; GFX8-NEXT: s_lshl_b32 s1, s1, s7
-; GFX8-NEXT: s_lshr_b32 s3, s3, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s3
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffff
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_and_b32 s3, s4, 15
-; GFX8-NEXT: s_and_b32 s5, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s6, s1
-; GFX8-NEXT: s_lshr_b32 s3, s5, s3
-; GFX8-NEXT: s_andn2_b32 s5, 15, s4
-; GFX8-NEXT: s_lshl_b32 s6, s0, 1
-; GFX8-NEXT: s_lshl_b32 s5, s6, s5
-; GFX8-NEXT: s_or_b32 s3, s5, s3
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100010
-; GFX8-NEXT: s_andn2_b32 s5, 15, s5
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: s_lshr_b32 s7, s2, 16
+; GFX8-NEXT: s_lshr_b32 s8, s4, 16
+; GFX8-NEXT: s_and_b32 s9, s4, 15
+; GFX8-NEXT: s_andn2_b32 s4, 15, s4
; GFX8-NEXT: s_lshl_b32 s0, s0, 1
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_bfe_u32 s4, s4, 0x40010
-; GFX8-NEXT: s_lshl_b32 s0, s0, s5
-; GFX8-NEXT: s_lshr_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s0, s0, s4
+; GFX8-NEXT: s_lshr_b32 s2, s2, s9
; GFX8-NEXT: s_or_b32 s0, s0, s2
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s3, s0
+; GFX8-NEXT: s_and_b32 s2, s8, 15
+; GFX8-NEXT: s_andn2_b32 s4, 15, s8
+; GFX8-NEXT: s_lshl_b32 s6, s6, 1
+; GFX8-NEXT: s_lshl_b32 s4, s6, s4
+; GFX8-NEXT: s_lshr_b32 s2, s7, s2
+; GFX8-NEXT: s_or_b32 s4, s4, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s4
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_lshr_b32 s2, s1, 16
+; GFX8-NEXT: s_lshr_b32 s4, s3, 16
+; GFX8-NEXT: s_lshr_b32 s6, s5, 16
+; GFX8-NEXT: s_and_b32 s7, s5, 15
+; GFX8-NEXT: s_andn2_b32 s5, 15, s5
+; GFX8-NEXT: s_lshl_b32 s1, s1, 1
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s1, s1, s5
+; GFX8-NEXT: s_lshr_b32 s3, s3, s7
+; GFX8-NEXT: s_or_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, s6, 15
+; GFX8-NEXT: s_andn2_b32 s5, 15, s6
+; GFX8-NEXT: s_lshl_b32 s2, s2, 1
+; GFX8-NEXT: s_lshl_b32 s2, s2, s5
+; GFX8-NEXT: s_lshr_b32 s3, s4, s3
+; GFX8-NEXT: s_or_b32 s2, s2, s3
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX11-LABEL: s_fshr_v4i16:
@@ -4608,57 +4858,92 @@ define <4 x half> @v_fshr_v4i16(<4 x i16> %lhs, <4 x i16> %rhs, <4 x i16> %amt)
; GFX6-LABEL: v_fshr_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX6-NEXT: v_and_b32_e32 v10, 0xffff0000, v3
-; GFX6-NEXT: v_or_b32_e32 v9, 16, v9
-; GFX6-NEXT: v_or_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v12, 15, v4
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v4
; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX6-NEXT: v_alignbit_b32 v8, v8, v10, v9
-; GFX6-NEXT: v_and_b32_e32 v9, 0xffff0000, v2
-; GFX6-NEXT: v_or_b32_e32 v7, 16, v7
-; GFX6-NEXT: v_alignbit_b32 v1, v1, v3, v5
-; GFX6-NEXT: v_or_b32_e32 v3, 16, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_alignbit_b32 v6, v6, v9, v7
-; GFX6-NEXT: v_alignbit_b32 v0, v0, v2, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 1, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v12, v2
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v10
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v10
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 1, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v4, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v8
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_xor_b32_e32 v4, -1, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v5
+; GFX6-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 1, v1
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v3
+; GFX6-NEXT: v_xor_b32_e32 v3, -1, v11
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 15, v11
+; GFX6-NEXT: v_and_b32_e32 v3, 15, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 1, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v9
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v6
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_fshr_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v7, 1
+; GFX8-NEXT: v_xor_b32_e32 v7, -1, v4
+; GFX8-NEXT: v_and_b32_e32 v6, 15, v4
+; GFX8-NEXT: v_and_b32_e32 v7, 15, v7
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, 1, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v7, v7, v8
+; GFX8-NEXT: v_lshrrev_b16_e32 v6, v6, v2
+; GFX8-NEXT: v_or_b32_e32 v6, v7, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, 15
; GFX8-NEXT: v_mov_b32_e32 v9, -1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v8, v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_xor_b32_sdwa v10, v5, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_sdwa v6, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v8, v10, v8
-; GFX8-NEXT: v_lshlrev_b16_sdwa v7, v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_xor_b32_sdwa v9, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v6, v8, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_sdwa v8, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v7, v9, v7
-; GFX8-NEXT: v_or_b32_sdwa v7, v7, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, 1, v1
-; GFX8-NEXT: v_xor_b32_e32 v8, -1, v5
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, v8, v1
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, v5, v3
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, 1, v0
-; GFX8-NEXT: v_xor_b32_e32 v3, -1, v4
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v3, v0
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, v4, v2
+; GFX8-NEXT: v_and_b32_sdwa v8, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v4, v4, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v10, 1
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v10, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, v4, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v2, v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_xor_b32_e32 v4, -1, v5
+; GFX8-NEXT: v_or_b32_sdwa v0, v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v2, 15, v5
+; GFX8-NEXT: v_and_b32_e32 v4, 15, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v6, 1, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, v4, v6
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v2, v3
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_and_b32_sdwa v4, v5, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_xor_b32_sdwa v5, v5, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v5, 15, v5
+; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, v5, v1
+; GFX8-NEXT: v_lshrrev_b16_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: v_fshr_v4i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir
index a9615553312d6..c286402649bec 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-abs.mir
@@ -302,8 +302,7 @@ body: |
; SI-NEXT: [[ABS:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG]]
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; SI-NEXT: [[ABS1:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG1]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ABS1]], [[C1]](s32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ABS1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ABS]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -319,10 +318,9 @@ body: |
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[ABS:%[0-9]+]]:_(s16) = G_ABS [[TRUNC]]
; VI-NEXT: [[ABS1:%[0-9]+]]:_(s16) = G_ABS [[TRUNC1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ABS]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ABS1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ABS]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ABS1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -410,11 +408,10 @@ body: |
; SI-NEXT: [[ABS2:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG2]]
; SI-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[ABS3:%[0-9]+]]:_(s32) = G_ABS [[SEXT_INREG3]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ABS1]], [[C1]](s32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ABS1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ABS]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ABS3]], [[C1]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ABS3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ABS2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -438,15 +435,14 @@ body: |
; VI-NEXT: [[ABS1:%[0-9]+]]:_(s16) = G_ABS [[TRUNC1]]
; VI-NEXT: [[ABS2:%[0-9]+]]:_(s16) = G_ABS [[TRUNC2]]
; VI-NEXT: [[ABS3:%[0-9]+]]:_(s16) = G_ABS [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ABS]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ABS1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ABS]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ABS1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ABS2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ABS3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ABS2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ABS3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
index ad251622d2b29..28b9772b42f46 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-add.mir
@@ -151,11 +151,10 @@ body: |
; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -176,10 +175,9 @@ body: |
; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -314,16 +312,15 @@ body: |
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR2]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
; GFX6-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR1]], [[LSHR3]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C1]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ADD3]], [[C1]]
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -357,15 +354,14 @@ body: |
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[TRUNC7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ADD3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
index 7f6a66950b518..32383d6fb5457 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-and.mir
@@ -453,9 +453,9 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV3]](<2 x s16>), [[UV4]](<2 x s16>)
; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV6]](<2 x s16>), [[UV7]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -496,60 +496,59 @@ body: |
bb.0:
; CHECK-LABEL: name: test_and_v5s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CHECK-NEXT: [[AND:%[0-9]+]]:_(<4 x s16>) = G_AND [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AND]](<4 x s16>)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C2]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL4]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL5]]
; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL6]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL7]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>)
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(<4 x s16>) = G_AND [[CONCAT_VECTORS2]], [[CONCAT_VECTORS3]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[AND1]](<4 x s16>)
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C2]](i32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C3]]
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST4]], [[C2]]
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C3]]
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C3]]
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[BITCAST5]], [[C2]]
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[BITCAST10]], [[C2]]
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND4]], [[C1]](i32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL9]]
; CHECK-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL10]]
; CHECK-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL11]]
; CHECK-NEXT: [[BITCAST14:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS4:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST11]](<2 x s16>), [[BITCAST12]](<2 x s16>), [[BITCAST13]](<2 x s16>), [[BITCAST14]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
index ea03954f4f9c4..024889bc85a9f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-anyext.mir
@@ -638,13 +638,14 @@ body: |
; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i16) = G_SHL [[AND3]], [[C4]](i16)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i16) = G_OR [[AND3]], [[SHL2]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i16) = COPY [[OR2]](i16)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](i16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](i16)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](i16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](i16)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C5]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](i16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[COPY1]](i16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](i16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[COPY1]](i16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C5]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR3]](i32), [[OR4]](i32)
; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
@@ -682,13 +683,14 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s448) = G_MERGE_VALUES [[MV]](s64), [[DEF]](s64), [[DEF]](s64), [[DEF]](s64), [[DEF]](s64), [[DEF]](s64), [[DEF]](s64)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
index 8283b18568b4a..1a7af6f5fd067 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ashr.mir
@@ -711,10 +711,10 @@ body: |
; SI-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG]], [[AND]](s32)
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[LSHR1]](i32)
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C2]]
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -735,10 +735,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -775,11 +774,10 @@ body: |
; SI-NEXT: [[ASHR:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG]], [[UV]](s32)
; SI-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; SI-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG1]], [[UV1]](s32)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C1]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -799,10 +797,9 @@ body: |
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -858,18 +855,18 @@ body: |
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; SI-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[AND1]](s32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C2]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ASHR2]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
@@ -898,19 +895,18 @@ body: |
; VI-NEXT: [[ASHR:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[ASHR2:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC2]], [[TRUNC5]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -985,15 +981,15 @@ body: |
; SI-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG2]], [[AND1]](s32)
; SI-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SEXT_INREG3]], [[LSHR3]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C2]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C1]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ASHR2]], [[C2]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[ASHR3]], [[C2]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -1027,15 +1023,14 @@ body: |
; VI-NEXT: [[ASHR1:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC1]], [[TRUNC5]](s16)
; VI-NEXT: [[ASHR2:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC2]], [[TRUNC6]](s16)
; VI-NEXT: [[ASHR3:%[0-9]+]]:_(s16) = G_ASHR [[TRUNC3]], [[TRUNC7]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ASHR3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ASHR3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
index 2db4ffb61e6c4..6efd646679157 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
@@ -472,10 +472,10 @@ body: |
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CHECK-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
@@ -496,15 +496,15 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
@@ -640,10 +640,10 @@ body: |
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[ADD3]], [[C]]
; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -779,10 +779,10 @@ body: |
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[ADD7]], [[C]]
; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C3]](s16)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s16) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR5]](s16)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C4]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL6]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -843,15 +843,14 @@ body: |
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC5]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[TRUNC6]]
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[TRUNC7]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ADD3]](s16)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
@@ -940,33 +939,33 @@ body: |
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[UV1]](s32)
; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV3]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s8) = COPY [[TRUNC1]](s8)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s8) = COPY [[TRUNC2]](s8)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s8) = COPY [[TRUNC3]](s8)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV4]], [[C]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL3]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[UV6]], [[C]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C2]](s32)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[UV7]], [[C]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[UV4]], [[C]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[UV5]], [[C]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL3]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[UV6]], [[C]]
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[UV7]], [[C]]
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C3]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
@@ -1013,14 +1012,14 @@ body: |
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C]]
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C1]](s16)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C2]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[MV]](s64)
@@ -1110,44 +1109,45 @@ body: |
; CHECK-NEXT: [[TRUNC15:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR13]](s32)
; CHECK-NEXT: [[COPY16:%[0-9]+]]:_(s16) = COPY [[TRUNC15]](s16)
; CHECK-NEXT: [[ADD15:%[0-9]+]]:_(s16) = G_ADD [[TRUNC15]], [[COPY16]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C7]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ADD3]](s16)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C7]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[ADD4]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[ADD5]](s16)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[ADD4]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[ADD5]](s16)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C7]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[ADD6]](s16)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[ADD7]](s16)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[ADD6]](s16)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[ADD7]](s16)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C7]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[ADD8]](s16)
- ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[ADD9]](s16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[ADD8]](s16)
+ ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(i32) = G_ZEXT [[ADD9]](s16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT9]], [[C7]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL4]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[ADD10]](s16)
- ; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[ADD11]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[ADD10]](s16)
+ ; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(i32) = G_ZEXT [[ADD11]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT11]], [[C7]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL5]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; CHECK-NEXT: [[ZEXT12:%[0-9]+]]:_(s32) = G_ZEXT [[ADD12]](s16)
- ; CHECK-NEXT: [[ZEXT13:%[0-9]+]]:_(s32) = G_ZEXT [[ADD13]](s16)
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT13]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT12:%[0-9]+]]:_(i32) = G_ZEXT [[ADD12]](s16)
+ ; CHECK-NEXT: [[ZEXT13:%[0-9]+]]:_(i32) = G_ZEXT [[ADD13]](s16)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT13]], [[C7]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[ZEXT12]], [[SHL6]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; CHECK-NEXT: [[ZEXT14:%[0-9]+]]:_(s32) = G_ZEXT [[ADD14]](s16)
- ; CHECK-NEXT: [[ZEXT15:%[0-9]+]]:_(s32) = G_ZEXT [[ADD15]](s16)
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT15]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT14:%[0-9]+]]:_(i32) = G_ZEXT [[ADD14]](s16)
+ ; CHECK-NEXT: [[ZEXT15:%[0-9]+]]:_(i32) = G_ZEXT [[ADD15]](s16)
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ZEXT15]], [[C7]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[ZEXT14]], [[SHL7]]
; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -1196,35 +1196,34 @@ body: |
; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR2]](i32)
; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s4) = G_TRUNC [[BITCAST3]](i32)
; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s4) = G_TRUNC [[LSHR3]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C3]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND4]], [[C5]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C6]](s32)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C7]](s32)
- ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C8]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C3]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C1]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C1]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND4]], [[C]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C1]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 20
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C5]](i32)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[BITCAST3]], [[C1]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C6]](i32)
+ ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[LSHR3]], [[C1]]
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 28
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C7]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s4) = COPY [[TRUNC]](s4)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s4) = COPY [[TRUNC1]](s4)
@@ -1234,27 +1233,27 @@ body: |
; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s4) = COPY [[TRUNC5]](s4)
; CHECK-NEXT: [[COPY7:%[0-9]+]]:_(s4) = COPY [[TRUNC6]](s4)
; CHECK-NEXT: [[COPY8:%[0-9]+]]:_(s4) = COPY [[TRUNC7]](s4)
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C1]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C2]](s32)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL7]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C1]]
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C3]](s32)
- ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C1]]
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C4]](s32)
- ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[SHL9]]
- ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[BITCAST6]], [[C1]]
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[AND12]], [[C5]](s32)
- ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
- ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C1]]
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C6]](s32)
- ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
- ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[BITCAST7]], [[C1]]
- ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C7]](s32)
- ; CHECK-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[OR11]], [[SHL12]]
- ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C1]]
- ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C8]](s32)
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[BITCAST4]], [[C1]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C1]]
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL7]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[BITCAST5]], [[C1]]
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C3]](i32)
+ ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[OR7]], [[SHL8]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[LSHR5]], [[C1]]
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C4]](i32)
+ ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[OR8]], [[SHL9]]
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(i32) = G_AND [[BITCAST6]], [[C1]]
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[AND12]], [[C]](i32)
+ ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[OR9]], [[SHL10]]
+ ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C1]]
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[AND13]], [[C5]](i32)
+ ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[OR10]], [[SHL11]]
+ ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(i32) = G_AND [[BITCAST7]], [[C1]]
+ ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[AND14]], [[C6]](i32)
+ ; CHECK-NEXT: [[OR12:%[0-9]+]]:_(i32) = G_OR [[OR11]], [[SHL12]]
+ ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(i32) = G_AND [[LSHR7]], [[C1]]
+ ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[AND15]], [[C7]](i32)
; CHECK-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[OR12]], [[SHL13]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[OR6]](s32), [[OR13]](s32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<2 x s32>)
@@ -1407,41 +1406,41 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr8_vgpr9_vgpr10_vgpr11
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<4 x s32>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; CHECK-NEXT: [[UV4:%[0-9]+]]:_(s32), [[UV5:%[0-9]+]]:_(s32), [[UV6:%[0-9]+]]:_(s32), [[UV7:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV4]], [[C]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL3]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[UV6]], [[C]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C2]](s32)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[UV7]], [[C]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[UV4]], [[C]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[UV5]], [[C]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL3]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[UV6]], [[C]]
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[UV7]], [[C]]
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C3]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
; CHECK-NEXT: [[UV8:%[0-9]+]]:_(s32), [[UV9:%[0-9]+]]:_(s32), [[UV10:%[0-9]+]]:_(s32), [[UV11:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY2]](<4 x s32>)
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[UV8]], [[C]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[UV9]], [[C]]
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL6]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[UV10]], [[C]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C2]](s32)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[UV11]], [[C]]
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[UV8]], [[C]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[UV9]], [[C]]
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL6]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[UV10]], [[C]]
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[UV11]], [[C]]
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C3]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<3 x s32>)
@@ -1497,17 +1496,17 @@ body: |
; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[ADD5]], [[C]]
; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C1]](s16)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND4]], [[SHL2]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C2]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT2]], [[C2]](s32)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C2]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT1]], [[SHL5]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -1544,20 +1543,20 @@ body: |
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s16) = G_LSHR [[ADD]], [[C]](s16)
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[ADD1]], [[C]](s16)
; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[ADD2]], [[C]](s16)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR]](s16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR]](s16)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR1]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR1]](s16)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR2]](s16)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -1691,14 +1690,14 @@ body: |
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C]]
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C1]](s16)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C2]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
@@ -1733,13 +1732,13 @@ body: |
; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC23]], [[C]]
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C1]](s16)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR6]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR7]](s16)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C2]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[OR8]](s16)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[OR9]](s16)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C2]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64)
@@ -1863,19 +1862,19 @@ body: |
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s8) = G_TRUNC [[UV1]](s32)
; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s8) = G_TRUNC [[UV2]](s32)
; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s8) = G_TRUNC [[UV3]](s32)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s8) = COPY [[TRUNC1]](s8)
@@ -1885,15 +1884,15 @@ body: |
; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s8) = G_TRUNC [[UV5]](s32)
; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s8) = G_TRUNC [[UV6]](s32)
; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s8) = G_TRUNC [[UV7]](s32)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV4]], [[C]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[AND4]], [[SHL3]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[UV6]], [[C]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND6]], [[C2]](s32)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[UV7]], [[C]]
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[UV4]], [[C]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[UV5]], [[C]]
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL3]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[UV6]], [[C]]
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND6]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[UV7]], [[C]]
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C3]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
; CHECK-NEXT: [[COPY6:%[0-9]+]]:_(s8) = COPY [[TRUNC1]](s8)
@@ -1907,15 +1906,15 @@ body: |
; CHECK-NEXT: [[TRUNC9:%[0-9]+]]:_(s8) = G_TRUNC [[UV9]](s32)
; CHECK-NEXT: [[TRUNC10:%[0-9]+]]:_(s8) = G_TRUNC [[UV10]](s32)
; CHECK-NEXT: [[TRUNC11:%[0-9]+]]:_(s8) = G_TRUNC [[UV11]](s32)
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[UV8]], [[C]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[UV9]], [[C]]
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL6]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[UV10]], [[C]]
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C2]](s32)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[UV11]], [[C]]
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[UV8]], [[C]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[UV9]], [[C]]
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL6]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[UV10]], [[C]]
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[UV11]], [[C]]
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C3]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
; CHECK-NEXT: [[COPY13:%[0-9]+]]:_(s8) = COPY [[TRUNC]](s8)
; CHECK-NEXT: [[COPY14:%[0-9]+]]:_(s8) = COPY [[TRUNC1]](s8)
@@ -1929,15 +1928,15 @@ body: |
; CHECK-NEXT: [[COPY22:%[0-9]+]]:_(s8) = COPY [[TRUNC9]](s8)
; CHECK-NEXT: [[COPY23:%[0-9]+]]:_(s8) = COPY [[TRUNC10]](s8)
; CHECK-NEXT: [[COPY24:%[0-9]+]]:_(s8) = COPY [[TRUNC11]](s8)
- ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[UV12]], [[C]]
- ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[UV13]], [[C]]
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C1]](s32)
- ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[AND12]], [[SHL9]]
- ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[UV14]], [[C]]
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[AND14]], [[C2]](s32)
- ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
- ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[UV15]], [[C]]
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(i32) = G_AND [[UV12]], [[C]]
+ ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(i32) = G_AND [[UV13]], [[C]]
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND13]], [[C1]](i32)
+ ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND12]], [[SHL9]]
+ ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(i32) = G_AND [[UV14]], [[C]]
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[AND14]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[OR9]], [[SHL10]]
+ ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(i32) = G_AND [[UV15]], [[C]]
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[AND15]], [[C3]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[OR2]](s32), [[OR5]](s32), [[OR8]](s32), [[OR11]](s32)
; CHECK-NEXT: S_ENDPGM 0, implicit [[BUILD_VECTOR]](<4 x s32>)
@@ -2156,25 +2155,25 @@ body: |
; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC29]], [[C]]
; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C1]](s16)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL7]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL8]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C2]](i32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL9]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR4]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR5]](s16)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C2]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL10]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[OR6]](s16)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[OR7]](s16)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C2]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL11]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -2299,59 +2298,60 @@ body: |
; CHECK-NEXT: [[LSHR15:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC9]], [[C1]](s16)
; CHECK-NEXT: [[LSHR16:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC10]], [[C1]](s16)
; CHECK-NEXT: [[LSHR17:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC11]], [[C1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C2]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT]], [[C]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C2]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR2]](s16)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C3]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR3]](s16)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV4]], [[C2]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT2]], [[C]](s32)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV4]], [[C2]]
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR4]](s16)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C3]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR5]](s16)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR5]](s16)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C3]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV5]], [[C2]]
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR8]](s16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT4]], [[C]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV5]], [[C2]]
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR8]](s16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT4]], [[C3]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR9]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR9]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C3]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR6]], [[SHL5]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV6]], [[C2]]
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR10]](s16)
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT6]], [[C]](s32)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV6]], [[C2]]
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR10]](s16)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT6]], [[C3]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL6]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR11]](s16)
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR11]](s16)
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C3]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[LSHR7]], [[SHL7]]
; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV7]], [[C2]]
- ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR14]](s16)
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT8]], [[C]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[UV7]], [[C2]]
+ ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR14]](s16)
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[ZEXT8]], [[C3]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL8]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR15]](s16)
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR15]](s16)
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[ZEXT9]], [[C3]](i32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[LSHR12]], [[SHL9]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[UV8]], [[C2]]
- ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR16]](s16)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT10]], [[C]](s32)
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[UV8]], [[C2]]
+ ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR16]](s16)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ZEXT10]], [[C3]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[AND5]], [[SHL10]]
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
- ; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR17]](s16)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR17]](s16)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ZEXT11]], [[C3]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[LSHR13]], [[SHL11]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<24 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>), [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>), [[BITCAST10]](<2 x s16>), [[BITCAST11]](<2 x s16>)
@@ -2409,14 +2409,13 @@ body: |
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C1]]
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND7]], [[C2]](s16)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[SHL3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
@@ -2452,13 +2451,13 @@ body: |
; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s16) = G_AND [[TRUNC15]], [[C1]]
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s16) = G_SHL [[AND15]], [[C2]](s16)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s16) = G_OR [[AND14]], [[SHL9]]
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](s16)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR6]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR7]](s16)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL10]]
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](s16)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](s16)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[OR8]](s16)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[OR9]](s16)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[ZEXT6]], [[SHL11]]
; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](s32), [[OR11]](s32)
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(i32) = G_BITCAST [[UV8]](<2 x s16>)
@@ -2493,13 +2492,13 @@ body: |
; CHECK-NEXT: [[AND23:%[0-9]+]]:_(s16) = G_AND [[TRUNC23]], [[C1]]
; CHECK-NEXT: [[SHL15:%[0-9]+]]:_(s16) = G_SHL [[AND23]], [[C2]](s16)
; CHECK-NEXT: [[OR15:%[0-9]+]]:_(s16) = G_OR [[AND22]], [[SHL15]]
- ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR12]](s16)
- ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR13]](s16)
- ; CHECK-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[OR12]](s16)
+ ; CHECK-NEXT: [[ZEXT9:%[0-9]+]]:_(i32) = G_ZEXT [[OR13]](s16)
+ ; CHECK-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[ZEXT9]], [[C]](i32)
; CHECK-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[ZEXT8]], [[SHL16]]
- ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR14]](s16)
- ; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR15]](s16)
- ; CHECK-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[OR14]](s16)
+ ; CHECK-NEXT: [[ZEXT11:%[0-9]+]]:_(i32) = G_ZEXT [[OR15]](s16)
+ ; CHECK-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[ZEXT11]], [[C]](i32)
; CHECK-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[ZEXT10]], [[SHL17]]
; CHECK-NEXT: [[MV2:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR16]](s32), [[OR17]](s32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s64>) = G_BUILD_VECTOR [[MV]](s64), [[MV1]](s64), [[MV2]](s64)
@@ -2545,25 +2544,24 @@ body: |
; CHECK-NEXT: [[ADD5:%[0-9]+]]:_(s16) = G_ADD [[LSHR4]], [[LSHR4]]
; CHECK-NEXT: [[ADD6:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[TRUNC3]]
; CHECK-NEXT: [[ADD7:%[0-9]+]]:_(s16) = G_ADD [[LSHR5]], [[LSHR5]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ADD3]](s16)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[ADD4]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[ADD5]](s16)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[ADD4]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[ADD5]](s16)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[ADD6]](s16)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[ADD7]](s16)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[ADD6]](s16)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[ADD7]](s16)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL3]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -2643,15 +2641,14 @@ body: |
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[OR1]], [[OR1]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[OR2]], [[OR2]]
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[OR3]], [[OR3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ADD3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir
index 8874dc7ac4aa9..644df9ab85f5f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitreverse.mir
@@ -99,7 +99,7 @@ body: |
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITREVERSE]], [[C1]](s32)
; CHECK-NEXT: [[BITREVERSE1:%[0-9]+]]:_(s32) = G_BITREVERSE [[LSHR]]
; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[BITREVERSE1]], [[C1]](s32)
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
index c79842ecb99ef..e2d9a2de5ec56 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bswap.mir
@@ -92,10 +92,9 @@ body: |
; GFX7-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[COPY3]](s32)
; GFX7-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](s32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[TRUNC3]], [[TRUNC2]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX7-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX7-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX7-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -161,23 +160,22 @@ body: |
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX8-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[BSWAP:%[0-9]+]]:_(<2 x s16>) = G_BSWAP [[BITCAST]]
; GFX8-NEXT: [[BSWAP1:%[0-9]+]]:_(<2 x s16>) = G_BSWAP [[BITCAST1]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[BSWAP]](<2 x s16>)
- ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C3]](i32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[BSWAP1]](<2 x s16>)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](i32)
; GFX8-NEXT: $vgpr1 = COPY [[LSHR]](i32)
@@ -247,15 +245,14 @@ body: |
; GFX7-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[COPY7]](s32)
; GFX7-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
; GFX7-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[TRUNC7]], [[TRUNC6]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX7-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX7-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; GFX7-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX7-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
+ ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX7-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX7-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector-trunc.mir
index f8d001b6de097..a59bbd02661f7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector-trunc.mir
@@ -24,11 +24,11 @@ body: |
; PREGFX8-NEXT: {{ $}}
; PREGFX8-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; PREGFX8-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; PREGFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; PREGFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; PREGFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; PREGFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; PREGFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; PREGFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; PREGFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; PREGFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; PREGFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; PREGFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; PREGFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; PREGFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; PREGFX8-NEXT: S_NOP 0, implicit [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.s16.mir
index 0c4e8b753137e..b569189a9246d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-build-vector.s16.mir
@@ -16,11 +16,11 @@ body: |
; GFX78-NEXT: {{ $}}
; GFX78-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; GFX78-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX78-NEXT: S_NOP 0, implicit [[BITCAST]](<2 x s16>)
@@ -54,21 +54,21 @@ body: |
; GFX78-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; GFX78-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX78-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX78-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX78-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX78-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX78-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX78-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -112,16 +112,16 @@ body: |
; GFX78-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; GFX78-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; GFX78-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX78-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX78-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX78-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -168,31 +168,31 @@ body: |
; GFX78-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; GFX78-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
; GFX78-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX78-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX78-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX78-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX78-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C1]](i32)
; GFX78-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; GFX78-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; GFX78-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX78-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX78-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C1]](i32)
; GFX78-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL4]]
; GFX78-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
; GFX78-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<10 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -249,41 +249,41 @@ body: |
; GFX78-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
; GFX78-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
; GFX78-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX78-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX78-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
- ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[COPY5]], [[C]]
+ ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX78-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX78-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
- ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[COPY6]], [[C]]
+ ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C1]](i32)
; GFX78-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; GFX78-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; GFX78-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C1]](i32)
; GFX78-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL4]]
; GFX78-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX78-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX78-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX78-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C1]](i32)
; GFX78-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL5]]
; GFX78-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; GFX78-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
- ; GFX78-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
- ; GFX78-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND12:%[0-9]+]]:_(i32) = G_AND [[COPY5]], [[C]]
+ ; GFX78-NEXT: [[AND13:%[0-9]+]]:_(i32) = G_AND [[COPY6]], [[C]]
+ ; GFX78-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND13]], [[C1]](i32)
; GFX78-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND12]], [[SHL6]]
; GFX78-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
; GFX78-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<14 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
@@ -351,26 +351,26 @@ body: |
; GFX78-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
; GFX78-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
; GFX78-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX78-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX78-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
- ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[COPY5]], [[C]]
+ ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX78-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX78-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
- ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
- ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[COPY6]], [[C]]
+ ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[COPY7]], [[C]]
+ ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C1]](i32)
; GFX78-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; GFX78-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; GFX78-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -446,46 +446,46 @@ body: |
; GFX78-NEXT: [[COPY13:%[0-9]+]]:_(s32) = COPY $vgpr13
; GFX78-NEXT: [[COPY14:%[0-9]+]]:_(s32) = COPY $vgpr14
; GFX78-NEXT: [[COPY15:%[0-9]+]]:_(s32) = COPY $vgpr15
- ; GFX78-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX78-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; GFX78-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX78-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX78-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; GFX78-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; GFX78-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX78-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX78-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX78-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; GFX78-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; GFX78-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX78-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX78-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[COPY4]], [[C]]
- ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[COPY5]], [[C]]
- ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY4]], [[C]]
+ ; GFX78-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[COPY5]], [[C]]
+ ; GFX78-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX78-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX78-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[COPY6]], [[C]]
- ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[COPY7]], [[C]]
- ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[COPY6]], [[C]]
+ ; GFX78-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[COPY7]], [[C]]
+ ; GFX78-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C1]](i32)
; GFX78-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; GFX78-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; GFX78-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[COPY8]], [[C]]
- ; GFX78-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[COPY9]], [[C]]
- ; GFX78-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[COPY8]], [[C]]
+ ; GFX78-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[COPY9]], [[C]]
+ ; GFX78-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C1]](i32)
; GFX78-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL4]]
; GFX78-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX78-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[COPY10]], [[C]]
- ; GFX78-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[COPY11]], [[C]]
- ; GFX78-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[COPY10]], [[C]]
+ ; GFX78-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[COPY11]], [[C]]
+ ; GFX78-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C1]](i32)
; GFX78-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL5]]
; GFX78-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; GFX78-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[COPY12]], [[C]]
- ; GFX78-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[COPY13]], [[C]]
- ; GFX78-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND13]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND12:%[0-9]+]]:_(i32) = G_AND [[COPY12]], [[C]]
+ ; GFX78-NEXT: [[AND13:%[0-9]+]]:_(i32) = G_AND [[COPY13]], [[C]]
+ ; GFX78-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND13]], [[C1]](i32)
; GFX78-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND12]], [[SHL6]]
; GFX78-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; GFX78-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[COPY14]], [[C]]
- ; GFX78-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[COPY15]], [[C]]
- ; GFX78-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND15]], [[C1]](s32)
+ ; GFX78-NEXT: [[AND14:%[0-9]+]]:_(i32) = G_AND [[COPY14]], [[C]]
+ ; GFX78-NEXT: [[AND15:%[0-9]+]]:_(i32) = G_AND [[COPY15]], [[C]]
+ ; GFX78-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND15]], [[C1]](i32)
; GFX78-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND14]], [[SHL7]]
; GFX78-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; GFX78-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<16 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir
index bbdddb83c10a8..3bc8a5666ac71 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-poison.mir
@@ -151,11 +151,10 @@ body: |
; CHECK-NEXT: [[AMDGPU_FFBH_U32_:%[0-9]+]]:_(i32) = G_AMDGPU_FFBH_U32 [[SHL]](i32)
; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[AMDGPU_FFBH_U32_1:%[0-9]+]]:_(i32) = G_AMDGPU_FFBH_U32 [[SHL1]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32_]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32_1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[AMDGPU_FFBH_U32_]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[AMDGPU_FFBH_U32_1]], [[C1]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
index cc17961ed068d..5b27d1da25518 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz.mir
@@ -177,9 +177,10 @@ body: |
; CHECK-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[AMDGPU_FFBH_U32_1]], [[C2]]
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = nuw G_SUB [[UMIN1]], [[C3]]
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB1]](s32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C3]](s32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C4]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C4]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
index 98d7d12e25e10..251b0eb309911 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctpop.mir
@@ -194,8 +194,7 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTPOP]](s32)
; CHECK-NEXT: [[CTPOP1:%[0-9]+]]:_(s32) = G_CTPOP [[LSHR]](i32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTPOP1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY2]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir
index 4408131de634a..939db590e50d9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz-zero-poison.mir
@@ -148,8 +148,7 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_POISON]](s32)
; CHECK-NEXT: [[CTTZ_ZERO_POISON1:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[LSHR]](i32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_POISON1]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY2]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
index 3354f734b3ca7..4d22a8657fadf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-cttz.mir
@@ -167,8 +167,7 @@ body: |
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[LSHR]], [[C1]]
; CHECK-NEXT: [[CTTZ_ZERO_POISON1:%[0-9]+]]:_(s32) = G_CTTZ_ZERO_POISON [[OR1]](s32)
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[CTTZ_ZERO_POISON1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY2]], [[C2]](s32)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY2]], [[C]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[COPY1]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
index 7b206c9bd8370..9841a2bac74d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-extract.mir
@@ -950,9 +950,9 @@ body: |
bb.0:
; CHECK-LABEL: name: extract_v2s16_v3s16_offset0
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -967,9 +967,9 @@ body: |
bb.0:
; CHECK-LABEL: name: extract_v2s16_v5s16_offset0
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -1545,15 +1545,14 @@ body: |
; CHECK-NEXT: [[OR18:%[0-9]+]]:_(i16) = G_OR [[OR17]], [[SHL5]]
; CHECK-NEXT: [[OR19:%[0-9]+]]:_(i16) = G_OR [[OR18]], [[SHL6]]
; CHECK-NEXT: [[OR20:%[0-9]+]]:_(i16) = G_OR [[OR19]], [[SHL7]]
- ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C11]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](i16)
- ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[ZEXT]], [[C12]](s32)
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C11]]
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR7]](i16)
+ ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[ZEXT]], [[C2]](i32)
; CHECK-NEXT: [[OR21:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL12]]
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR14]](i16)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR20]](i16)
- ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[ZEXT2]], [[C12]](s32)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR14]](i16)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR20]](i16)
+ ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[ZEXT2]], [[C2]](i32)
; CHECK-NEXT: [[OR22:%[0-9]+]]:_(i32) = G_OR [[ZEXT1]], [[SHL13]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR21]](i32), [[OR22]](i32)
; CHECK-NEXT: [[DEF1:%[0-9]+]]:_(s64) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
index 62e7e3e8da591..e791c31e89e14 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fabs.mir
@@ -266,9 +266,9 @@ body: |
bb.0:
; SI-LABEL: name: test_fabs_v3s16
- ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
@@ -276,28 +276,27 @@ body: |
; SI-NEXT: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST]]
; SI-NEXT: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST1]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FABS]](<2 x s16>)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FABS1]](<2 x s16>)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C2]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL1]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST3]], [[C2]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C1]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL3]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: S_NOP 0, implicit [[CONCAT_VECTORS]](<6 x s16>)
;
; VI-LABEL: name: test_fabs_v3s16
- ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
@@ -305,19 +304,18 @@ body: |
; VI-NEXT: [[FABS:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST]]
; VI-NEXT: [[FABS1:%[0-9]+]]:_(<2 x s16>) = G_FABS [[BITCAST1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FABS]](<2 x s16>)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FABS1]](<2 x s16>)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C2]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL1]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST3]], [[C2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL3]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
index 7aab6577aa083..d75a820c67fa7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fadd.mir
@@ -345,10 +345,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -369,10 +368,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -427,18 +425,17 @@ body: |
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; SI-NEXT: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -466,18 +463,17 @@ body: |
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC3]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC4]]
; VI-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[TRUNC2]], [[TRUNC5]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -561,15 +557,14 @@ body: |
; SI-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
; SI-NEXT: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT6]], [[FPEXT7]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -603,15 +598,14 @@ body: |
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[TRUNC3]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FADD3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
index fba60c5c584c4..e5beee915fdd3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcanonicalize.mir
@@ -248,10 +248,9 @@ body: |
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; SI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT1]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -267,10 +266,9 @@ body: |
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FCANONICALIZE:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC]]
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FCANONICALIZE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FCANONICALIZE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -369,15 +367,14 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s32) = G_FCANONICALIZE [[FPEXT3]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCANONICALIZE3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -401,15 +398,14 @@ body: |
; VI-NEXT: [[FCANONICALIZE1:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC2]]
; VI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FCANONICALIZE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FCANONICALIZE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FCANONICALIZE3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FCANONICALIZE2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FCANONICALIZE3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
index 1cb6999ef529b..826132d6e9010 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fceil.mir
@@ -183,10 +183,9 @@ body: |
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; SI-NEXT: [[FCEIL1:%[0-9]+]]:_(s32) = G_FCEIL [[FPEXT1]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCEIL1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -206,10 +205,9 @@ body: |
; CI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; CI-NEXT: [[FCEIL1:%[0-9]+]]:_(s32) = G_FCEIL [[FPEXT1]]
; CI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FCEIL1]](s32)
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -225,10 +223,9 @@ body: |
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FCEIL:%[0-9]+]]:_(s16) = G_FCEIL [[TRUNC]]
; VI-NEXT: [[FCEIL1:%[0-9]+]]:_(s16) = G_FCEIL [[TRUNC1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FCEIL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FCEIL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FCEIL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FCEIL1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcopysign.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcopysign.mir
index 282cc63f53585..bb4a5467cfba3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcopysign.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcopysign.mir
@@ -532,13 +532,13 @@ body: |
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32768
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 32768
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[AND:%[0-9]+]]:_(<2 x i16>) = G_AND [[COPY]], [[BITCAST1]]
@@ -551,13 +551,13 @@ body: |
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 32768
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 32768
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 32767
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 32767
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x i16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[AND:%[0-9]+]]:_(<2 x i16>) = G_AND [[COPY]], [[BITCAST1]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
index 0e6a129e5cce9..1970f01423c48 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fcos.mir
@@ -324,10 +324,9 @@ body: |
; SI-NEXT: [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL1]](s32)
; SI-NEXT: [[INT3:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[INT2]](s32)
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -348,10 +347,9 @@ body: |
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[C1]]
; VI-NEXT: [[INT2:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL1]](s16)
; VI-NEXT: [[INT3:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[INT2]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INT1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INT3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -485,15 +483,14 @@ body: |
; SI-NEXT: [[INT6:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL3]](s32)
; SI-NEXT: [[INT7:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[INT6]](s32)
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT7]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -526,15 +523,14 @@ body: |
; VI-NEXT: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[C1]]
; VI-NEXT: [[INT6:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL3]](s16)
; VI-NEXT: [[INT7:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.cos), [[INT6]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INT1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INT3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[INT5]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[INT7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[INT5]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[INT7]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
index 8cc3a0b6b5242..dd34cb1b5c72b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fdiv.mir
@@ -1783,10 +1783,9 @@ body: |
; SI-NEXT: [[INT12:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fmas), [[FMA9]](f32), [[FMA6]](f32), [[FMA8]](f32), [[INT10]](s1)
; SI-NEXT: [[INT13:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[INT12]](f32), [[FPEXT3]](s32), [[FPEXT2]](s32)
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT13]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -1842,10 +1841,9 @@ body: |
; VI-NEXT: [[FADD7:%[0-9]+]]:_(f32) = G_FADD [[BITCAST5]], [[FADD5]]
; VI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = G_FPTRUNC [[FADD7]](f32)
; VI-NEXT: [[INT3:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[FPTRUNC1]](f16), [[TRUNC3]](s16), [[TRUNC1]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INT1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INT3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
@@ -2414,15 +2412,14 @@ body: |
; SI-NEXT: [[INT26:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fmas), [[FMA19]](f32), [[FMA16]](f32), [[FMA18]](f32), [[INT24]](s1)
; SI-NEXT: [[INT27:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[INT26]](f32), [[FPEXT7]](s32), [[FPEXT6]](s32)
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT27]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -2525,15 +2522,14 @@ body: |
; VI-NEXT: [[FADD15:%[0-9]+]]:_(f32) = G_FADD [[BITCAST11]], [[FADD13]]
; VI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(f16) = G_FPTRUNC [[FADD15]](f32)
; VI-NEXT: [[INT7:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.div.fixup), [[FPTRUNC3]](f16), [[TRUNC7]](s16), [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INT1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INT3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[INT5]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[INT7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[INT5]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[INT7]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST12]](<2 x s16>), [[BITCAST13]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir
index b5d1044b79d51..67ba424d49b93 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp.mir
@@ -618,10 +618,10 @@ body: |
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[FPEXT1]], [[C]]
; GFX6-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[FMUL1]](f32)
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT1]](f32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -639,10 +639,10 @@ body: |
; GFX8-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[FPEXT1]], [[C]]
; GFX8-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[FMUL1]](f32)
; GFX8-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT1]](f32)
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
index effff2be3a9cc..7fea8b2fd8f32 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fexp2.mir
@@ -316,10 +316,9 @@ body: |
; GFX6-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC1]](s16)
; GFX6-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.exp2), [[FPEXT1]](f32)
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT1]](f32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -335,10 +334,9 @@ body: |
; GFX8-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; GFX8-NEXT: [[FEXP2_:%[0-9]+]]:_(s16) = G_FEXP2 [[TRUNC]]
; GFX8-NEXT: [[FEXP2_1:%[0-9]+]]:_(s16) = G_FEXP2 [[TRUNC1]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FEXP2_]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FEXP2_1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FEXP2_]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FEXP2_1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
index b85895237c6dd..d1c80c196cb60 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ffloor.mir
@@ -344,10 +344,9 @@ body: |
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; SI-NEXT: [[FFLOOR1:%[0-9]+]]:_(s32) = G_FFLOOR [[FPEXT1]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FFLOOR1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -363,10 +362,9 @@ body: |
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FFLOOR:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC]]
; VI-NEXT: [[FFLOOR1:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FFLOOR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FFLOOR1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -470,15 +468,14 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FFLOOR3:%[0-9]+]]:_(s32) = G_FFLOOR [[FPEXT3]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FFLOOR3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -502,15 +499,14 @@ body: |
; VI-NEXT: [[FFLOOR1:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC1]]
; VI-NEXT: [[FFLOOR2:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC2]]
; VI-NEXT: [[FFLOOR3:%[0-9]+]]:_(s16) = G_FFLOOR [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FFLOOR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FFLOOR1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FFLOOR3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FFLOOR2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FFLOOR3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
index de56199d8dcbc..c4e2431f4e2e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog.mir
@@ -249,10 +249,9 @@ body: |
; CHECK-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT1]](f32)
; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[INT1]], [[C1]]
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](f32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
index 5e85ab7a744ba..f156ca856ec9b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-flog10.mir
@@ -249,10 +249,9 @@ body: |
; CHECK-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.log), [[FPEXT1]](f32)
; CHECK-NEXT: [[FMUL1:%[0-9]+]]:_(f32) = G_FMUL [[INT1]], [[C1]]
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](f32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
index e38bdb4848976..a0fb7f5ac4db8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fma.mir
@@ -402,10 +402,9 @@ body: |
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; SI-NEXT: [[FMA1:%[0-9]+]]:_(s32) = G_FMA [[FPEXT3]], [[FPEXT4]], [[FPEXT5]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -431,10 +430,9 @@ body: |
; VI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; VI-NEXT: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC2]], [[TRUNC4]]
; VI-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC3]], [[TRUNC5]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMA]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMA1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -503,18 +501,17 @@ body: |
; SI-NEXT: [[FPEXT8:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC8]](s16)
; SI-NEXT: [[FMA2:%[0-9]+]]:_(s32) = G_FMA [[FPEXT6]], [[FPEXT7]], [[FPEXT8]]
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA2]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>), [[BITCAST8]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -550,18 +547,17 @@ body: |
; VI-NEXT: [[FMA:%[0-9]+]]:_(s16) = G_FMA [[TRUNC]], [[TRUNC3]], [[TRUNC6]]
; VI-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC4]], [[TRUNC7]]
; VI-NEXT: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[TRUNC2]], [[TRUNC5]], [[TRUNC8]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMA]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMA1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMA2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMA2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>), [[BITCAST8]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -663,15 +659,14 @@ body: |
; SI-NEXT: [[FPEXT11:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC11]](s16)
; SI-NEXT: [[FMA3:%[0-9]+]]:_(s32) = G_FMA [[FPEXT9]], [[FPEXT10]], [[FPEXT11]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMA3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -715,15 +710,14 @@ body: |
; VI-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC5]], [[TRUNC9]]
; VI-NEXT: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[TRUNC2]], [[TRUNC6]], [[TRUNC10]]
; VI-NEXT: [[FMA3:%[0-9]+]]:_(s16) = G_FMA [[TRUNC3]], [[TRUNC7]], [[TRUNC11]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMA]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMA1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMA2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMA3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMA2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMA3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
index bbabd060065d2..f7c5924a38b1a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmad.s16.mir
@@ -118,10 +118,9 @@ body: |
; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; GFX6-NEXT: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT6]], [[FPEXT7]]
; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -147,10 +146,9 @@ body: |
; GFX7-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR2]](i32)
; GFX7-NEXT: [[FMAD:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC]], [[TRUNC2]], [[TRUNC4]]
; GFX7-NEXT: [[FMAD1:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC1]], [[TRUNC3]], [[TRUNC5]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD1]](s16)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAD]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAD1]](s16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -264,15 +262,14 @@ body: |
; GFX6-NEXT: [[FPEXT15:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC11]](s16)
; GFX6-NEXT: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT14]], [[FPEXT15]]
; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC7]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC5]](s16)
+ ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC7]](s16)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -316,15 +313,14 @@ body: |
; GFX7-NEXT: [[FMAD1:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC1]], [[TRUNC5]], [[TRUNC9]]
; GFX7-NEXT: [[FMAD2:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC2]], [[TRUNC6]], [[TRUNC10]]
; GFX7-NEXT: [[FMAD3:%[0-9]+]]:_(s16) = G_FMAD [[TRUNC3]], [[TRUNC7]], [[TRUNC11]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD1]](s16)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAD]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAD1]](s16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD2]](s16)
- ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMAD3]](s16)
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMAD2]](s16)
+ ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMAD3]](s16)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -568,10 +564,9 @@ body: |
; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; GFX6-NEXT: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT6]], [[FPEXT7]]
; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -599,10 +594,9 @@ body: |
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[FMUL]], [[TRUNC4]]
; GFX7-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
; GFX7-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[FMUL1]], [[TRUNC5]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -685,10 +679,9 @@ body: |
; GFX6-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC5]](s16)
; GFX6-NEXT: [[FADD1:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT6]], [[FPEXT7]]
; GFX6-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -716,10 +709,9 @@ body: |
; GFX7-NEXT: [[FADD:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL]], [[TRUNC4]]
; GFX7-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC1]], [[TRUNC3]]
; GFX7-NEXT: [[FADD1:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL1]], [[TRUNC5]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX7-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -833,15 +825,14 @@ body: |
; GFX6-NEXT: [[FPEXT15:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC11]](s16)
; GFX6-NEXT: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT14]], [[FPEXT15]]
; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC7]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC5]](s16)
+ ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC7]](s16)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -889,15 +880,14 @@ body: |
; GFX7-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[FMUL2]], [[TRUNC10]]
; GFX7-NEXT: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[TRUNC7]]
; GFX7-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[FMUL3]], [[TRUNC11]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FADD3]](s16)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -1034,15 +1024,14 @@ body: |
; GFX6-NEXT: [[FPEXT15:%[0-9]+]]:_(s32) = nnan G_FPEXT [[TRUNC11]](s16)
; GFX6-NEXT: [[FADD3:%[0-9]+]]:_(s32) = nnan G_FADD [[FPEXT14]], [[FPEXT15]]
; GFX6-NEXT: [[FPTRUNC7:%[0-9]+]]:_(s16) = nnan G_FPTRUNC [[FADD3]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC7]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC5]](s16)
+ ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC7]](s16)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -1090,15 +1079,14 @@ body: |
; GFX7-NEXT: [[FADD2:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL2]], [[TRUNC10]]
; GFX7-NEXT: [[FMUL3:%[0-9]+]]:_(s16) = nnan G_FMUL [[TRUNC3]], [[TRUNC7]]
; GFX7-NEXT: [[FADD3:%[0-9]+]]:_(s16) = nnan G_FADD [[FMUL3]], [[TRUNC11]]
- ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX7-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX7-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX7-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX7-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX7-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX7-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; GFX7-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FADD3]](s16)
+ ; GFX7-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX7-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX7-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX7-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
index 4ff37b77bf111..f6ab150e89c09 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmaxnum.mir
@@ -417,10 +417,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -445,10 +444,9 @@ body: |
; VI-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -505,18 +503,17 @@ body: |
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; SI-NEXT: [[FMAXNUM_IEEE2:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT4]], [[FPEXT5]]
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE2]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -550,18 +547,17 @@ body: |
; VI-NEXT: [[FCANONICALIZE4:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC2]]
; VI-NEXT: [[FCANONICALIZE5:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC5]]
; VI-NEXT: [[FMAXNUM_IEEE2:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE4]], [[FCANONICALIZE5]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -649,15 +645,14 @@ body: |
; SI-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
; SI-NEXT: [[FMAXNUM_IEEE3:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT6]], [[FPEXT7]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -699,15 +694,14 @@ body: |
; VI-NEXT: [[FCANONICALIZE6:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[FCANONICALIZE7:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC7]]
; VI-NEXT: [[FMAXNUM_IEEE3:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE6]], [[FCANONICALIZE7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -1016,10 +1010,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[C]](s16)
; SI-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMAXNUM_IEEE [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMAXNUM_IEEE1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1040,10 +1033,9 @@ body: |
; VI-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[C]]
; VI-NEXT: [[FMAXNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
index 60828e4cda495..7fd1345167bbf 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fminnum.mir
@@ -417,10 +417,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -445,10 +444,9 @@ body: |
; VI-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -505,18 +503,17 @@ body: |
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; SI-NEXT: [[FMINNUM_IEEE2:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT4]], [[FPEXT5]]
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE2]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -550,18 +547,17 @@ body: |
; VI-NEXT: [[FCANONICALIZE4:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC2]]
; VI-NEXT: [[FCANONICALIZE5:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC5]]
; VI-NEXT: [[FMINNUM_IEEE2:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE4]], [[FCANONICALIZE5]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -649,15 +645,14 @@ body: |
; SI-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
; SI-NEXT: [[FMINNUM_IEEE3:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT6]], [[FPEXT7]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -699,15 +694,14 @@ body: |
; VI-NEXT: [[FCANONICALIZE6:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC3]]
; VI-NEXT: [[FCANONICALIZE7:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC7]]
; VI-NEXT: [[FMINNUM_IEEE3:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE6]], [[FCANONICALIZE7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -1016,10 +1010,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[C]](s16)
; SI-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(s32) = G_FMINNUM_IEEE [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMINNUM_IEEE1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1040,10 +1033,9 @@ body: |
; VI-NEXT: [[FCANONICALIZE2:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC1]]
; VI-NEXT: [[FCANONICALIZE3:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[C]]
; VI-NEXT: [[FMINNUM_IEEE1:%[0-9]+]]:_(s16) = G_FMINNUM_IEEE [[FCANONICALIZE2]], [[FCANONICALIZE3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMINNUM_IEEE1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
index 26fe4e2bbf70f..6b36842b0e9d1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fmul.mir
@@ -344,10 +344,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -368,10 +367,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -427,18 +425,17 @@ body: |
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC5]](s16)
; SI-NEXT: [[FMUL2:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT4]], [[FPEXT5]]
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL2]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -466,18 +463,17 @@ body: |
; VI-NEXT: [[FMUL:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC]], [[TRUNC3]]
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC4]]
; VI-NEXT: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC2]], [[TRUNC5]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -561,15 +557,14 @@ body: |
; SI-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
; SI-NEXT: [[FMUL3:%[0-9]+]]:_(s32) = G_FMUL [[FPEXT6]], [[FPEXT7]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -603,15 +598,14 @@ body: |
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[FMUL2:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMUL3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMUL3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
index d2b67ac781372..6e23efa190d52 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fneg.mir
@@ -264,9 +264,9 @@ body: |
bb.0:
; SI-LABEL: name: test_fneg_v3s16
- ; SI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
@@ -274,19 +274,18 @@ body: |
; SI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST]]
; SI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST1]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C2]]
; SI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32)
; SI-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s32>)
;
; VI-LABEL: name: test_fneg_v3s16
- ; VI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
@@ -294,12 +293,11 @@ body: |
; VI-NEXT: [[FNEG:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST]]
; VI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[BITCAST1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG]](<2 x s16>)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C2]](i32)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[FNEG1]](<2 x s16>)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C3]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C2]]
; VI-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND]](s32), [[LSHR]](i32), [[AND1]](s32)
; VI-NEXT: S_NOP 0, implicit [[BUILD_VECTOR]](<3 x s32>)
;
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
index ac6a14525c12b..f9abf16bf55e1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fpow.mir
@@ -481,10 +481,9 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(f32) = G_SELECT [[FCMP1]](s1), [[C4]], [[C5]]
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = G_FMUL [[INT5]], [[SELECT3]]
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FMUL1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -570,10 +569,9 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(f32) = nnan nsz G_SELECT [[FCMP1]](s1), [[C4]], [[C5]]
; GFX6-NEXT: [[FMUL1:%[0-9]+]]:_(s32) = nnan nsz G_FMUL [[INT5]], [[SELECT3]]
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = nnan nsz G_FPTRUNC [[FMUL1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
index e42ad0d04ecd0..0368684fc6e4c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fptrunc.mir
@@ -70,15 +70,14 @@ body: |
; CHECK-NEXT: [[UV:%[0-9]+]]:_(f32), [[UV1:%[0-9]+]]:_(f32) = G_UNMERGE_VALUES [[COPY]](<2 x f32>)
; CHECK-NEXT: [[FPTRUNC:%[0-9]+]]:_(f16) = G_FPTRUNC [[UV]](f32)
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = G_FPTRUNC [[UV1]](f32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](f16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](f16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](f16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](f16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x f16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[BITCAST1]](i32), [[LSHR]](i32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x f32>) = COPY $vgpr0_vgpr1
@@ -283,10 +282,11 @@ body: |
; CHECK-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[UV5]], [[C19]](s32)
; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR9]], [[C20]]
; CHECK-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[AND9]], [[SELECT7]]
- ; CHECK-NEXT: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[OR7]], [[C21]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[OR15]], [[C21]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C19]](s32)
+ ; CHECK-NEXT: [[C21:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[OR7]], [[C21]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[OR15]], [[C21]]
+ ; CHECK-NEXT: [[C22:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C22]](i32)
; CHECK-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL4]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR16]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -390,10 +390,10 @@ body: |
; CHECK-NEXT: [[FPTRUNC1:%[0-9]+]]:_(f16) = afn G_FPTRUNC [[FPTRUNC]](s32)
; CHECK-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s32) = afn G_FPTRUNC [[UV1]](f64)
; CHECK-NEXT: [[FPTRUNC3:%[0-9]+]]:_(f16) = afn G_FPTRUNC [[FPTRUNC2]](s32)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](f16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](f16)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](f16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](f16)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
index 329fa748a89d7..2a4f2e385b994 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-freeze.mir
@@ -540,24 +540,23 @@ body: |
; CHECK-LABEL: name: test_freeze_v3s16
; CHECK: [[COPY:%[0-9]+]]:_(<3 x s32>) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<3 x s32>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<4 x s16>) = G_FREEZE [[CONCAT_VECTORS]]
; CHECK-NEXT: [[UV3:%[0-9]+]]:_(<2 x s16>), [[UV4:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[FREEZE]](<4 x s16>)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C3]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV4]](<2 x s16>)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[BITCAST2]](i32), [[LSHR]](i32), [[BITCAST3]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
@@ -590,31 +589,30 @@ body: |
; CHECK-LABEL: name: test_freeze_v5s16
; CHECK: [[COPY:%[0-9]+]]:_(<5 x s32>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32), [[UV4:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY]](<5 x s32>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV4]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[UV4]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
; CHECK-NEXT: [[FREEZE:%[0-9]+]]:_(<6 x s16>) = G_FREEZE [[CONCAT_VECTORS]]
; CHECK-NEXT: [[UV5:%[0-9]+]]:_(<2 x s16>), [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[FREEZE]](<6 x s16>)
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[UV5]](<2 x s16>)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C3]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST3]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV6]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C3]](i32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<5 x s32>) = G_BUILD_VECTOR [[BITCAST3]](i32), [[LSHR]](i32), [[BITCAST4]](i32), [[LSHR1]](i32), [[BITCAST5]](i32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4 = COPY [[BUILD_VECTOR]](<5 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
index 96a065ec80cfb..99704529828eb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshl.mir
@@ -271,10 +271,9 @@ body: |
; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[LSHR5]], [[ZEXT3]](s32)
; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR6]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
- ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -315,10 +314,9 @@ body: |
; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[C3]](s16)
; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[LSHR5]], [[AND3]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL1]], [[LSHR6]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -725,19 +723,18 @@ body: |
; SI-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[LSHR7]], [[ZEXT5]](s32)
; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR8]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC7]], [[TRUNC8]]
- ; SI-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL3]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C5]], [[C]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL4]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C5]], [[C]](i32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL5]]
; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -795,19 +792,18 @@ body: |
; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[C3]](s16)
; VI-NEXT: [[LSHR8:%[0-9]+]]:_(s16) = G_LSHR [[LSHR7]], [[AND5]](s16)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL2]], [[LSHR8]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL5]]
; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -960,15 +956,14 @@ body: |
; SI-NEXT: [[LSHR13:%[0-9]+]]:_(s32) = G_LSHR [[LSHR12]], [[ZEXT7]](s32)
; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR13]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
- ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT9]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT9:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT9]], [[C]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL4]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT11]], [[C]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL5]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -1039,15 +1034,14 @@ body: |
; VI-NEXT: [[LSHR12:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC7]], [[C3]](s16)
; VI-NEXT: [[LSHR13:%[0-9]+]]:_(s16) = G_LSHR [[LSHR12]], [[AND7]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR13]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
index 07af830f04f4a..67c3455e75802 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fshr.mir
@@ -233,10 +233,9 @@ body: |
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[ZEXT3]](s32)
; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
- ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL4]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -277,10 +276,9 @@ body: |
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[SHL2]], [[AND3]](s16)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[AND2]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR4]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST3]](<2 x s16>)
@@ -681,19 +679,18 @@ body: |
; SI-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[AND7]], [[ZEXT5]](s32)
; SI-NEXT: [[TRUNC8:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR5]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC7]], [[TRUNC8]]
- ; SI-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL6]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT8:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C5]], [[C]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT8]], [[SHL7]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
- ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL8]]
+ ; SI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[C5]], [[C]](i32)
+ ; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL8]]
; SI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -751,19 +748,18 @@ body: |
; VI-NEXT: [[SHL5:%[0-9]+]]:_(s16) = G_SHL [[SHL4]], [[AND5]](s16)
; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC5]], [[AND4]](s16)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL5]], [[LSHR5]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL6]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL7]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
- ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL8]]
+ ; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
+ ; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL8]]
; VI-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr1 = COPY [[BITCAST7]](<2 x s16>)
@@ -883,10 +879,9 @@ body: |
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[ZEXT3]](s32)
; SI-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR4]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC4]], [[TRUNC5]]
- ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL4]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
@@ -922,9 +917,9 @@ body: |
; SI-NEXT: [[LSHR9:%[0-9]+]]:_(s32) = G_LSHR [[LSHR6]], [[ZEXT9]](s32)
; SI-NEXT: [[TRUNC11:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR9]](s32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[TRUNC10]], [[TRUNC11]]
- ; SI-NEXT: [[ZEXT10:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
- ; SI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT11]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT10:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; SI-NEXT: [[ZEXT11:%[0-9]+]]:_(i32) = G_ZEXT [[OR4]](s16)
+ ; SI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[ZEXT11]], [[C]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT10]], [[SHL9]]
; SI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST7]](<2 x s16>)
@@ -969,10 +964,9 @@ body: |
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[SHL2]], [[AND3]](s16)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[AND2]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL3]], [[LSHR4]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
@@ -1001,9 +995,9 @@ body: |
; VI-NEXT: [[SHL8:%[0-9]+]]:_(s16) = G_SHL [[SHL7]], [[AND7]](s16)
; VI-NEXT: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC9]], [[AND6]](s16)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s16) = disjoint G_OR [[SHL8]], [[LSHR9]]
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
- ; VI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR4]](s16)
+ ; VI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL9]]
; VI-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST7]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
index 13b81b37f3236..248dbcf94f7ce 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsin.mir
@@ -324,10 +324,9 @@ body: |
; SI-NEXT: [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL1]](s32)
; SI-NEXT: [[INT3:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[INT2]](s32)
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT3]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -348,10 +347,9 @@ body: |
; VI-NEXT: [[FMUL1:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC1]], [[C1]]
; VI-NEXT: [[INT2:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL1]](s16)
; VI-NEXT: [[INT3:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[INT2]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INT1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INT3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -485,15 +483,14 @@ body: |
; SI-NEXT: [[INT6:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL3]](s32)
; SI-NEXT: [[INT7:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[INT6]](s32)
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT7]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -526,15 +523,14 @@ body: |
; VI-NEXT: [[FMUL3:%[0-9]+]]:_(s16) = G_FMUL [[TRUNC3]], [[C1]]
; VI-NEXT: [[INT6:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.fract), [[FMUL3]](s16)
; VI-NEXT: [[INT7:%[0-9]+]]:_(s16) = G_INTRINSIC intrinsic(@llvm.amdgcn.sin), [[INT6]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INT1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INT3]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INT1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INT3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[INT5]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[INT7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[INT5]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[INT7]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
index 27b227470b0ad..44e74ef72fb34 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsqrt.mir
@@ -410,10 +410,9 @@ body: |
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC1]](s16)
; SI-NEXT: [[INT1:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sqrt), [[FPEXT1]](f32)
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT1]](f32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -429,10 +428,9 @@ body: |
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[FSQRT:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC]]
; VI-NEXT: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FSQRT]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FSQRT1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -536,15 +534,14 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(f32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[INT3:%[0-9]+]]:_(f32) = G_INTRINSIC intrinsic(@llvm.amdgcn.sqrt), [[FPEXT3]](f32)
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[INT3]](f32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -568,15 +565,14 @@ body: |
; VI-NEXT: [[FSQRT1:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC1]]
; VI-NEXT: [[FSQRT2:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC2]]
; VI-NEXT: [[FSQRT3:%[0-9]+]]:_(s16) = G_FSQRT [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FSQRT]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FSQRT1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FSQRT3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FSQRT2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FSQRT3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
index bae375a55d3d5..22b3b16779440 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-fsub.mir
@@ -393,10 +393,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -418,10 +417,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -479,18 +477,17 @@ body: |
; SI-NEXT: [[FPEXT5:%[0-9]+]]:_(s32) = G_FPEXT [[FNEG1]](s16)
; SI-NEXT: [[FADD2:%[0-9]+]]:_(s32) = G_FADD [[FPEXT4]], [[FPEXT5]]
; SI-NEXT: [[FPTRUNC2:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD2]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -519,18 +516,17 @@ body: |
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC4]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[FSUB:%[0-9]+]]:_(s16) = G_FSUB [[TRUNC2]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FSUB]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FSUB]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -602,10 +598,9 @@ body: |
; SI-NEXT: [[FPEXT3:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC3]](s16)
; SI-NEXT: [[FADD1:%[0-9]+]]:_(s32) = G_FADD [[FPEXT2]], [[FPEXT3]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
@@ -625,9 +620,9 @@ body: |
; SI-NEXT: [[FPEXT7:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC7]](s16)
; SI-NEXT: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT6]], [[FPEXT7]]
; SI-NEXT: [[FPTRUNC3:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
- ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC3]](s16)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; SI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC3]](s16)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -652,10 +647,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[FADD:%[0-9]+]]:_(s16) = G_FADD [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[FNEG1:%[0-9]+]]:_(<2 x s16>) = G_FNEG [[UV3]]
@@ -669,9 +663,9 @@ body: |
; VI-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR3]](i32)
; VI-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[TRUNC4]], [[TRUNC6]]
; VI-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[TRUNC5]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FADD3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def-s1025.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def-s1025.mir
index 415da5dd940f4..f5de5ce398f46 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def-s1025.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def-s1025.mir
@@ -7,196 +7,196 @@ name: test_implicit_def_s1025
body: |
bb.0:
; TAHITI-LABEL: name: test_implicit_def_s1025
- ; TAHITI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; TAHITI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; TAHITI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; TAHITI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; TAHITI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; TAHITI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C2]](s32)
- ; TAHITI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; TAHITI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
- ; TAHITI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C3]](s32)
- ; TAHITI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
- ; TAHITI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; TAHITI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C4]](s32)
- ; TAHITI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
- ; TAHITI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; TAHITI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C5]](s32)
- ; TAHITI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; TAHITI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
- ; TAHITI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C6]](s32)
- ; TAHITI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
- ; TAHITI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
- ; TAHITI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C7]](s32)
- ; TAHITI-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
- ; TAHITI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; TAHITI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C8]](s32)
- ; TAHITI-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
- ; TAHITI-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
- ; TAHITI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C9]](s32)
- ; TAHITI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
- ; TAHITI-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; TAHITI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C10]](s32)
- ; TAHITI-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[SHL9]]
- ; TAHITI-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
- ; TAHITI-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C11]](s32)
- ; TAHITI-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
- ; TAHITI-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
- ; TAHITI-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C12]](s32)
- ; TAHITI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
- ; TAHITI-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
- ; TAHITI-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C13]](s32)
- ; TAHITI-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[OR11]], [[SHL12]]
- ; TAHITI-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
- ; TAHITI-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C14]](s32)
- ; TAHITI-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[OR12]], [[SHL13]]
- ; TAHITI-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
- ; TAHITI-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C15]](s32)
- ; TAHITI-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[OR13]], [[SHL14]]
- ; TAHITI-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; TAHITI-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C16]](s32)
- ; TAHITI-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[OR14]], [[SHL15]]
- ; TAHITI-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
- ; TAHITI-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C17]](s32)
- ; TAHITI-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[OR15]], [[SHL16]]
- ; TAHITI-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 18
- ; TAHITI-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C18]](s32)
- ; TAHITI-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[OR16]], [[SHL17]]
- ; TAHITI-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 19
- ; TAHITI-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C19]](s32)
- ; TAHITI-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[OR17]], [[SHL18]]
- ; TAHITI-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; TAHITI-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C20]](s32)
- ; TAHITI-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[OR18]], [[SHL19]]
- ; TAHITI-NEXT: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 21
- ; TAHITI-NEXT: [[SHL20:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C21]](s32)
- ; TAHITI-NEXT: [[OR20:%[0-9]+]]:_(s32) = G_OR [[OR19]], [[SHL20]]
- ; TAHITI-NEXT: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 22
- ; TAHITI-NEXT: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C22]](s32)
- ; TAHITI-NEXT: [[OR21:%[0-9]+]]:_(s32) = G_OR [[OR20]], [[SHL21]]
- ; TAHITI-NEXT: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
- ; TAHITI-NEXT: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C23]](s32)
- ; TAHITI-NEXT: [[OR22:%[0-9]+]]:_(s32) = G_OR [[OR21]], [[SHL22]]
- ; TAHITI-NEXT: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; TAHITI-NEXT: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C24]](s32)
- ; TAHITI-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[OR22]], [[SHL23]]
- ; TAHITI-NEXT: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
- ; TAHITI-NEXT: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C25]](s32)
- ; TAHITI-NEXT: [[OR24:%[0-9]+]]:_(s32) = G_OR [[OR23]], [[SHL24]]
- ; TAHITI-NEXT: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 26
- ; TAHITI-NEXT: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C26]](s32)
- ; TAHITI-NEXT: [[OR25:%[0-9]+]]:_(s32) = G_OR [[OR24]], [[SHL25]]
- ; TAHITI-NEXT: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 27
- ; TAHITI-NEXT: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C27]](s32)
- ; TAHITI-NEXT: [[OR26:%[0-9]+]]:_(s32) = G_OR [[OR25]], [[SHL26]]
- ; TAHITI-NEXT: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; TAHITI-NEXT: [[SHL27:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C28]](s32)
- ; TAHITI-NEXT: [[OR27:%[0-9]+]]:_(s32) = G_OR [[OR26]], [[SHL27]]
- ; TAHITI-NEXT: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 29
- ; TAHITI-NEXT: [[SHL28:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C29]](s32)
- ; TAHITI-NEXT: [[OR28:%[0-9]+]]:_(s32) = G_OR [[OR27]], [[SHL28]]
- ; TAHITI-NEXT: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 30
- ; TAHITI-NEXT: [[SHL29:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C30]](s32)
- ; TAHITI-NEXT: [[OR29:%[0-9]+]]:_(s32) = G_OR [[OR28]], [[SHL29]]
- ; TAHITI-NEXT: [[C31:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
- ; TAHITI-NEXT: [[SHL30:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C31]](s32)
+ ; TAHITI: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; TAHITI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; TAHITI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
+ ; TAHITI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; TAHITI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
+ ; TAHITI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C2]](i32)
+ ; TAHITI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; TAHITI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 3
+ ; TAHITI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C3]](i32)
+ ; TAHITI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
+ ; TAHITI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; TAHITI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C4]](i32)
+ ; TAHITI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; TAHITI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 5
+ ; TAHITI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C5]](i32)
+ ; TAHITI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; TAHITI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
+ ; TAHITI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C6]](i32)
+ ; TAHITI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; TAHITI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; TAHITI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C7]](i32)
+ ; TAHITI-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[OR5]], [[SHL6]]
+ ; TAHITI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; TAHITI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C8]](i32)
+ ; TAHITI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; TAHITI-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 9
+ ; TAHITI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C9]](i32)
+ ; TAHITI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[OR7]], [[SHL8]]
+ ; TAHITI-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 10
+ ; TAHITI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C10]](i32)
+ ; TAHITI-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[OR8]], [[SHL9]]
+ ; TAHITI-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 11
+ ; TAHITI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C11]](i32)
+ ; TAHITI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[OR9]], [[SHL10]]
+ ; TAHITI-NEXT: [[C12:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; TAHITI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C12]](i32)
+ ; TAHITI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[OR10]], [[SHL11]]
+ ; TAHITI-NEXT: [[C13:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; TAHITI-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C13]](i32)
+ ; TAHITI-NEXT: [[OR12:%[0-9]+]]:_(i32) = G_OR [[OR11]], [[SHL12]]
+ ; TAHITI-NEXT: [[C14:%[0-9]+]]:_(i32) = G_CONSTANT i32 14
+ ; TAHITI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C14]](i32)
+ ; TAHITI-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[OR12]], [[SHL13]]
+ ; TAHITI-NEXT: [[C15:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; TAHITI-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C15]](i32)
+ ; TAHITI-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[OR13]], [[SHL14]]
+ ; TAHITI-NEXT: [[C16:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; TAHITI-NEXT: [[SHL15:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C16]](i32)
+ ; TAHITI-NEXT: [[OR15:%[0-9]+]]:_(i32) = G_OR [[OR14]], [[SHL15]]
+ ; TAHITI-NEXT: [[C17:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; TAHITI-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C17]](i32)
+ ; TAHITI-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[OR15]], [[SHL16]]
+ ; TAHITI-NEXT: [[C18:%[0-9]+]]:_(i32) = G_CONSTANT i32 18
+ ; TAHITI-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C18]](i32)
+ ; TAHITI-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[OR16]], [[SHL17]]
+ ; TAHITI-NEXT: [[C19:%[0-9]+]]:_(i32) = G_CONSTANT i32 19
+ ; TAHITI-NEXT: [[SHL18:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C19]](i32)
+ ; TAHITI-NEXT: [[OR18:%[0-9]+]]:_(i32) = G_OR [[OR17]], [[SHL18]]
+ ; TAHITI-NEXT: [[C20:%[0-9]+]]:_(i32) = G_CONSTANT i32 20
+ ; TAHITI-NEXT: [[SHL19:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C20]](i32)
+ ; TAHITI-NEXT: [[OR19:%[0-9]+]]:_(i32) = G_OR [[OR18]], [[SHL19]]
+ ; TAHITI-NEXT: [[C21:%[0-9]+]]:_(i32) = G_CONSTANT i32 21
+ ; TAHITI-NEXT: [[SHL20:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C21]](i32)
+ ; TAHITI-NEXT: [[OR20:%[0-9]+]]:_(i32) = G_OR [[OR19]], [[SHL20]]
+ ; TAHITI-NEXT: [[C22:%[0-9]+]]:_(i32) = G_CONSTANT i32 22
+ ; TAHITI-NEXT: [[SHL21:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C22]](i32)
+ ; TAHITI-NEXT: [[OR21:%[0-9]+]]:_(i32) = G_OR [[OR20]], [[SHL21]]
+ ; TAHITI-NEXT: [[C23:%[0-9]+]]:_(i32) = G_CONSTANT i32 23
+ ; TAHITI-NEXT: [[SHL22:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C23]](i32)
+ ; TAHITI-NEXT: [[OR22:%[0-9]+]]:_(i32) = G_OR [[OR21]], [[SHL22]]
+ ; TAHITI-NEXT: [[C24:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; TAHITI-NEXT: [[SHL23:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C24]](i32)
+ ; TAHITI-NEXT: [[OR23:%[0-9]+]]:_(i32) = G_OR [[OR22]], [[SHL23]]
+ ; TAHITI-NEXT: [[C25:%[0-9]+]]:_(i32) = G_CONSTANT i32 25
+ ; TAHITI-NEXT: [[SHL24:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C25]](i32)
+ ; TAHITI-NEXT: [[OR24:%[0-9]+]]:_(i32) = G_OR [[OR23]], [[SHL24]]
+ ; TAHITI-NEXT: [[C26:%[0-9]+]]:_(i32) = G_CONSTANT i32 26
+ ; TAHITI-NEXT: [[SHL25:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C26]](i32)
+ ; TAHITI-NEXT: [[OR25:%[0-9]+]]:_(i32) = G_OR [[OR24]], [[SHL25]]
+ ; TAHITI-NEXT: [[C27:%[0-9]+]]:_(i32) = G_CONSTANT i32 27
+ ; TAHITI-NEXT: [[SHL26:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C27]](i32)
+ ; TAHITI-NEXT: [[OR26:%[0-9]+]]:_(i32) = G_OR [[OR25]], [[SHL26]]
+ ; TAHITI-NEXT: [[C28:%[0-9]+]]:_(i32) = G_CONSTANT i32 28
+ ; TAHITI-NEXT: [[SHL27:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C28]](i32)
+ ; TAHITI-NEXT: [[OR27:%[0-9]+]]:_(i32) = G_OR [[OR26]], [[SHL27]]
+ ; TAHITI-NEXT: [[C29:%[0-9]+]]:_(i32) = G_CONSTANT i32 29
+ ; TAHITI-NEXT: [[SHL28:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C29]](i32)
+ ; TAHITI-NEXT: [[OR28:%[0-9]+]]:_(i32) = G_OR [[OR27]], [[SHL28]]
+ ; TAHITI-NEXT: [[C30:%[0-9]+]]:_(i32) = G_CONSTANT i32 30
+ ; TAHITI-NEXT: [[SHL29:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C30]](i32)
+ ; TAHITI-NEXT: [[OR29:%[0-9]+]]:_(i32) = G_OR [[OR28]], [[SHL29]]
+ ; TAHITI-NEXT: [[C31:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
+ ; TAHITI-NEXT: [[SHL30:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C31]](i32)
; TAHITI-NEXT: [[OR30:%[0-9]+]]:_(s32) = G_OR [[OR29]], [[SHL30]]
; TAHITI-NEXT: $vgpr0 = COPY [[OR30]](s32)
;
; FIJI-LABEL: name: test_implicit_def_s1025
- ; FIJI: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; FIJI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; FIJI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; FIJI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; FIJI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; FIJI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C2]](s32)
- ; FIJI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; FIJI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
- ; FIJI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C3]](s32)
- ; FIJI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
- ; FIJI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; FIJI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C4]](s32)
- ; FIJI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
- ; FIJI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; FIJI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C5]](s32)
- ; FIJI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; FIJI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
- ; FIJI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C6]](s32)
- ; FIJI-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
- ; FIJI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
- ; FIJI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C7]](s32)
- ; FIJI-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
- ; FIJI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; FIJI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C8]](s32)
- ; FIJI-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
- ; FIJI-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
- ; FIJI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C9]](s32)
- ; FIJI-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
- ; FIJI-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; FIJI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C10]](s32)
- ; FIJI-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[SHL9]]
- ; FIJI-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
- ; FIJI-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C11]](s32)
- ; FIJI-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
- ; FIJI-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
- ; FIJI-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C12]](s32)
- ; FIJI-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
- ; FIJI-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
- ; FIJI-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C13]](s32)
- ; FIJI-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[OR11]], [[SHL12]]
- ; FIJI-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
- ; FIJI-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C14]](s32)
- ; FIJI-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[OR12]], [[SHL13]]
- ; FIJI-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
- ; FIJI-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C15]](s32)
- ; FIJI-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[OR13]], [[SHL14]]
- ; FIJI-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; FIJI-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C16]](s32)
- ; FIJI-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[OR14]], [[SHL15]]
- ; FIJI-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
- ; FIJI-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C17]](s32)
- ; FIJI-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[OR15]], [[SHL16]]
- ; FIJI-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 18
- ; FIJI-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C18]](s32)
- ; FIJI-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[OR16]], [[SHL17]]
- ; FIJI-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 19
- ; FIJI-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C19]](s32)
- ; FIJI-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[OR17]], [[SHL18]]
- ; FIJI-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; FIJI-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C20]](s32)
- ; FIJI-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[OR18]], [[SHL19]]
- ; FIJI-NEXT: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 21
- ; FIJI-NEXT: [[SHL20:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C21]](s32)
- ; FIJI-NEXT: [[OR20:%[0-9]+]]:_(s32) = G_OR [[OR19]], [[SHL20]]
- ; FIJI-NEXT: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 22
- ; FIJI-NEXT: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C22]](s32)
- ; FIJI-NEXT: [[OR21:%[0-9]+]]:_(s32) = G_OR [[OR20]], [[SHL21]]
- ; FIJI-NEXT: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
- ; FIJI-NEXT: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C23]](s32)
- ; FIJI-NEXT: [[OR22:%[0-9]+]]:_(s32) = G_OR [[OR21]], [[SHL22]]
- ; FIJI-NEXT: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; FIJI-NEXT: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C24]](s32)
- ; FIJI-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[OR22]], [[SHL23]]
- ; FIJI-NEXT: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
- ; FIJI-NEXT: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C25]](s32)
- ; FIJI-NEXT: [[OR24:%[0-9]+]]:_(s32) = G_OR [[OR23]], [[SHL24]]
- ; FIJI-NEXT: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 26
- ; FIJI-NEXT: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C26]](s32)
- ; FIJI-NEXT: [[OR25:%[0-9]+]]:_(s32) = G_OR [[OR24]], [[SHL25]]
- ; FIJI-NEXT: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 27
- ; FIJI-NEXT: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C27]](s32)
- ; FIJI-NEXT: [[OR26:%[0-9]+]]:_(s32) = G_OR [[OR25]], [[SHL26]]
- ; FIJI-NEXT: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; FIJI-NEXT: [[SHL27:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C28]](s32)
- ; FIJI-NEXT: [[OR27:%[0-9]+]]:_(s32) = G_OR [[OR26]], [[SHL27]]
- ; FIJI-NEXT: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 29
- ; FIJI-NEXT: [[SHL28:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C29]](s32)
- ; FIJI-NEXT: [[OR28:%[0-9]+]]:_(s32) = G_OR [[OR27]], [[SHL28]]
- ; FIJI-NEXT: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 30
- ; FIJI-NEXT: [[SHL29:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C30]](s32)
- ; FIJI-NEXT: [[OR29:%[0-9]+]]:_(s32) = G_OR [[OR28]], [[SHL29]]
- ; FIJI-NEXT: [[C31:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
- ; FIJI-NEXT: [[SHL30:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C31]](s32)
+ ; FIJI: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; FIJI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; FIJI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
+ ; FIJI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; FIJI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
+ ; FIJI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C2]](i32)
+ ; FIJI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; FIJI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 3
+ ; FIJI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C3]](i32)
+ ; FIJI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
+ ; FIJI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; FIJI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C4]](i32)
+ ; FIJI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; FIJI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 5
+ ; FIJI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C5]](i32)
+ ; FIJI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; FIJI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
+ ; FIJI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C6]](i32)
+ ; FIJI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; FIJI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; FIJI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C7]](i32)
+ ; FIJI-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[OR5]], [[SHL6]]
+ ; FIJI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; FIJI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C8]](i32)
+ ; FIJI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; FIJI-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 9
+ ; FIJI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C9]](i32)
+ ; FIJI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[OR7]], [[SHL8]]
+ ; FIJI-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 10
+ ; FIJI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C10]](i32)
+ ; FIJI-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[OR8]], [[SHL9]]
+ ; FIJI-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 11
+ ; FIJI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C11]](i32)
+ ; FIJI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[OR9]], [[SHL10]]
+ ; FIJI-NEXT: [[C12:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; FIJI-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C12]](i32)
+ ; FIJI-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[OR10]], [[SHL11]]
+ ; FIJI-NEXT: [[C13:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; FIJI-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C13]](i32)
+ ; FIJI-NEXT: [[OR12:%[0-9]+]]:_(i32) = G_OR [[OR11]], [[SHL12]]
+ ; FIJI-NEXT: [[C14:%[0-9]+]]:_(i32) = G_CONSTANT i32 14
+ ; FIJI-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C14]](i32)
+ ; FIJI-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[OR12]], [[SHL13]]
+ ; FIJI-NEXT: [[C15:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; FIJI-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C15]](i32)
+ ; FIJI-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[OR13]], [[SHL14]]
+ ; FIJI-NEXT: [[C16:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; FIJI-NEXT: [[SHL15:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C16]](i32)
+ ; FIJI-NEXT: [[OR15:%[0-9]+]]:_(i32) = G_OR [[OR14]], [[SHL15]]
+ ; FIJI-NEXT: [[C17:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; FIJI-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C17]](i32)
+ ; FIJI-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[OR15]], [[SHL16]]
+ ; FIJI-NEXT: [[C18:%[0-9]+]]:_(i32) = G_CONSTANT i32 18
+ ; FIJI-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C18]](i32)
+ ; FIJI-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[OR16]], [[SHL17]]
+ ; FIJI-NEXT: [[C19:%[0-9]+]]:_(i32) = G_CONSTANT i32 19
+ ; FIJI-NEXT: [[SHL18:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C19]](i32)
+ ; FIJI-NEXT: [[OR18:%[0-9]+]]:_(i32) = G_OR [[OR17]], [[SHL18]]
+ ; FIJI-NEXT: [[C20:%[0-9]+]]:_(i32) = G_CONSTANT i32 20
+ ; FIJI-NEXT: [[SHL19:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C20]](i32)
+ ; FIJI-NEXT: [[OR19:%[0-9]+]]:_(i32) = G_OR [[OR18]], [[SHL19]]
+ ; FIJI-NEXT: [[C21:%[0-9]+]]:_(i32) = G_CONSTANT i32 21
+ ; FIJI-NEXT: [[SHL20:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C21]](i32)
+ ; FIJI-NEXT: [[OR20:%[0-9]+]]:_(i32) = G_OR [[OR19]], [[SHL20]]
+ ; FIJI-NEXT: [[C22:%[0-9]+]]:_(i32) = G_CONSTANT i32 22
+ ; FIJI-NEXT: [[SHL21:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C22]](i32)
+ ; FIJI-NEXT: [[OR21:%[0-9]+]]:_(i32) = G_OR [[OR20]], [[SHL21]]
+ ; FIJI-NEXT: [[C23:%[0-9]+]]:_(i32) = G_CONSTANT i32 23
+ ; FIJI-NEXT: [[SHL22:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C23]](i32)
+ ; FIJI-NEXT: [[OR22:%[0-9]+]]:_(i32) = G_OR [[OR21]], [[SHL22]]
+ ; FIJI-NEXT: [[C24:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; FIJI-NEXT: [[SHL23:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C24]](i32)
+ ; FIJI-NEXT: [[OR23:%[0-9]+]]:_(i32) = G_OR [[OR22]], [[SHL23]]
+ ; FIJI-NEXT: [[C25:%[0-9]+]]:_(i32) = G_CONSTANT i32 25
+ ; FIJI-NEXT: [[SHL24:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C25]](i32)
+ ; FIJI-NEXT: [[OR24:%[0-9]+]]:_(i32) = G_OR [[OR23]], [[SHL24]]
+ ; FIJI-NEXT: [[C26:%[0-9]+]]:_(i32) = G_CONSTANT i32 26
+ ; FIJI-NEXT: [[SHL25:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C26]](i32)
+ ; FIJI-NEXT: [[OR25:%[0-9]+]]:_(i32) = G_OR [[OR24]], [[SHL25]]
+ ; FIJI-NEXT: [[C27:%[0-9]+]]:_(i32) = G_CONSTANT i32 27
+ ; FIJI-NEXT: [[SHL26:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C27]](i32)
+ ; FIJI-NEXT: [[OR26:%[0-9]+]]:_(i32) = G_OR [[OR25]], [[SHL26]]
+ ; FIJI-NEXT: [[C28:%[0-9]+]]:_(i32) = G_CONSTANT i32 28
+ ; FIJI-NEXT: [[SHL27:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C28]](i32)
+ ; FIJI-NEXT: [[OR27:%[0-9]+]]:_(i32) = G_OR [[OR26]], [[SHL27]]
+ ; FIJI-NEXT: [[C29:%[0-9]+]]:_(i32) = G_CONSTANT i32 29
+ ; FIJI-NEXT: [[SHL28:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C29]](i32)
+ ; FIJI-NEXT: [[OR28:%[0-9]+]]:_(i32) = G_OR [[OR27]], [[SHL28]]
+ ; FIJI-NEXT: [[C30:%[0-9]+]]:_(i32) = G_CONSTANT i32 30
+ ; FIJI-NEXT: [[SHL29:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C30]](i32)
+ ; FIJI-NEXT: [[OR29:%[0-9]+]]:_(i32) = G_OR [[OR28]], [[SHL29]]
+ ; FIJI-NEXT: [[C31:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
+ ; FIJI-NEXT: [[SHL30:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C31]](i32)
; FIJI-NEXT: [[OR30:%[0-9]+]]:_(s32) = G_OR [[OR29]], [[SHL30]]
; FIJI-NEXT: $vgpr0 = COPY [[OR30]](s32)
%0:_(s1025) = G_IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
index 7c414241091f0..e618259642308 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-implicit-def.mir
@@ -409,12 +409,12 @@ body: |
bb.0:
; CHECK-LABEL: name: test_implicit_def_v3s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -443,18 +443,18 @@ body: |
bb.0:
; CHECK-LABEL: name: test_implicit_def_v5s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -471,18 +471,18 @@ body: |
bb.0:
; CHECK-LABEL: name: test_implicit_def_v6s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
index 54a6234f4ed74..e093225291f51 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-insert.mir
@@ -1139,10 +1139,9 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1192,11 +1191,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -1221,10 +1220,9 @@ body: |
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1251,11 +1249,11 @@ body: |
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1281,11 +1279,11 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s16>) = COPY $vgpr0_vgpr1
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -1337,15 +1335,14 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr2
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1372,9 +1369,10 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1402,14 +1400,15 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C2]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL1]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -1437,10 +1436,9 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1464,11 +1462,11 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1494,10 +1492,9 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -1521,11 +1518,11 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -1551,10 +1548,9 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1582,14 +1578,13 @@ body: |
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST2]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -1614,10 +1609,9 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -1644,10 +1638,9 @@ body: |
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -1676,15 +1669,14 @@ body: |
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C1]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL1]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -1709,9 +1701,10 @@ body: |
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[UV1]](<2 x s16>)
@@ -1739,13 +1732,13 @@ body: |
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1769,9 +1762,10 @@ body: |
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY]](<4 x s16>)
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY1]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
index c6c01ece72bb6..0d02fadcb4c53 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-round.mir
@@ -634,10 +634,9 @@ body: |
; GFX6-NEXT: [[FPEXT13:%[0-9]+]]:_(s32) = G_FPEXT [[OR1]](s16)
; GFX6-NEXT: [[FADD3:%[0-9]+]]:_(s32) = G_FADD [[FPEXT12]], [[FPEXT13]]
; GFX6-NEXT: [[FPTRUNC5:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD3]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC5]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -674,10 +673,9 @@ body: |
; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i16) = G_AND [[TRUNC1]], [[C4]]
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(s16) = disjoint G_OR [[AND2]], [[AND3]]
; GFX8-NEXT: [[FADD1:%[0-9]+]]:_(s16) = G_FADD [[INTRINSIC_TRUNC1]], [[OR1]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -803,18 +801,17 @@ body: |
; GFX6-NEXT: [[FPEXT20:%[0-9]+]]:_(s32) = G_FPEXT [[OR2]](s16)
; GFX6-NEXT: [[FADD5:%[0-9]+]]:_(s32) = G_FADD [[FPEXT19]], [[FPEXT20]]
; GFX6-NEXT: [[FPTRUNC8:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD5]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC5]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC8]](s16)
- ; GFX6-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC8]](s16)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C]](i32)
; GFX6-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX6-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL1]]
+ ; GFX6-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL1]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -863,18 +860,17 @@ body: |
; GFX8-NEXT: [[AND5:%[0-9]+]]:_(i16) = G_AND [[TRUNC2]], [[C4]]
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(s16) = disjoint G_OR [[AND4]], [[AND5]]
; GFX8-NEXT: [[FADD2:%[0-9]+]]:_(s16) = G_FADD [[INTRINSIC_TRUNC2]], [[OR2]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; GFX8-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C6]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C]](i32)
; GFX8-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX8-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL1]]
+ ; GFX8-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -1042,15 +1038,14 @@ body: |
; GFX6-NEXT: [[FPEXT27:%[0-9]+]]:_(s32) = G_FPEXT [[OR3]](s16)
; GFX6-NEXT: [[FADD7:%[0-9]+]]:_(s32) = G_FADD [[FPEXT26]], [[FPEXT27]]
; GFX6-NEXT: [[FPTRUNC11:%[0-9]+]]:_(s16) = G_FPTRUNC [[FADD7]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC2]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC5]](s16)
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC2]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC5]](s16)
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX6-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC8]](s16)
- ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC11]](s16)
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C6]](s32)
+ ; GFX6-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC8]](s16)
+ ; GFX6-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC11]](s16)
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX6-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1111,15 +1106,14 @@ body: |
; GFX8-NEXT: [[AND7:%[0-9]+]]:_(i16) = G_AND [[TRUNC3]], [[C4]]
; GFX8-NEXT: [[OR3:%[0-9]+]]:_(s16) = disjoint G_OR [[AND6]], [[AND7]]
; GFX8-NEXT: [[FADD3:%[0-9]+]]:_(s16) = G_FADD [[INTRINSIC_TRUNC3]], [[OR3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FADD1]](s16)
- ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FADD2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FADD3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C6]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FADD2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FADD3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
index 131f8f98e2dd2..0219590946768 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-intrinsic-trunc.mir
@@ -176,10 +176,9 @@ body: |
; SI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; SI-NEXT: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT1]]
; SI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC1]](s32)
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -199,10 +198,9 @@ body: |
; CI-NEXT: [[FPEXT1:%[0-9]+]]:_(s32) = G_FPEXT [[TRUNC1]](s16)
; CI-NEXT: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s32) = G_INTRINSIC_TRUNC [[FPEXT1]]
; CI-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[INTRINSIC_TRUNC1]](s32)
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -218,10 +216,9 @@ body: |
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR]](i32)
; VI-NEXT: [[INTRINSIC_TRUNC:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC]]
; VI-NEXT: [[INTRINSIC_TRUNC1:%[0-9]+]]:_(s16) = G_INTRINSIC_TRUNC [[TRUNC1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[INTRINSIC_TRUNC]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[INTRINSIC_TRUNC1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[INTRINSIC_TRUNC]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[INTRINSIC_TRUNC1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
index dc7e10ce8598a..d5df0eb7c0af3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.load.2d.d16.ll
@@ -85,11 +85,11 @@ define amdgpu_ps <2 x half> @image_load_v2f16(<8 x i32> inreg %rsrc, i32 %s, i32
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -146,16 +146,16 @@ define amdgpu_ps <3 x half> @image_load_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -205,16 +205,16 @@ define amdgpu_ps <4 x half> @image_load_v4f16(<8 x i32> inreg %rsrc, i32 %s, i32
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<4 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -319,11 +319,11 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16(<8 x i32> inreg %rsrc, i32 %s,
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
@@ -384,16 +384,16 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16(<8 x i32> inreg %rsrc, i32 %s,
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x i32>)
; UNPACKED-NEXT: G_STORE [[UV3]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -454,16 +454,16 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16(<8 x i32> inreg %rsrc, i32 %s,
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<5 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 15, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<4 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32), [[UV4:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<5 x i32>)
; UNPACKED-NEXT: G_STORE [[UV4]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -536,11 +536,11 @@ define amdgpu_ps <2 x half> @image_load_v2f16_dmask_1000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -617,16 +617,16 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1100(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[DEF]], [[C]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[DEF]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -680,16 +680,16 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_1000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[DEF:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[DEF]], [[C]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[DEF]], [[C]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY [[AND1]](i32)
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY10]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -722,9 +722,9 @@ define amdgpu_ps <3 x half> @image_load_v3f16_dmask_0000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY $vgpr0
; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR]](i32)
@@ -776,16 +776,16 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1110(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -835,15 +835,15 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1100(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -892,11 +892,11 @@ define amdgpu_ps <4 x half> @image_load_v4f16_dmask_1000(<8 x i32> inreg %rsrc,
; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:_(i32) = COPY $vgpr1
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(i32) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 0, 0, 0 :: (dereferenceable load (f16), addrspace 8)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
@@ -1028,11 +1028,11 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_1000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
@@ -1088,11 +1088,11 @@ define amdgpu_ps <2 x half> @image_load_tfe_v2f16_dmask_0000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
@@ -1160,16 +1160,16 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1100(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
; UNPACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[DEF1]], [[C]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[DEF1]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -1237,16 +1237,16 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_1000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[DEF1]], [[C]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[DEF1]], [[C]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY [[AND1]](i32)
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY10]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -1314,16 +1314,16 @@ define amdgpu_ps <3 x half> @image_load_tfe_v3f16_dmask_0000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
; UNPACKED-NEXT: [[DEF1:%[0-9]+]]:_(i32) = G_IMPLICIT_DEF
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[DEF1]], [[C]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(s32) = COPY [[AND1]](s32)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY10]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[DEF1]], [[C]]
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:_(i32) = COPY [[AND1]](i32)
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[COPY10]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -1384,16 +1384,16 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1110(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<4 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 7, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<3 x f16>), align 8, addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<4 x i32>)
; UNPACKED-NEXT: G_STORE [[UV3]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -1454,15 +1454,15 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1100(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<3 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 3, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (<2 x f16>), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32), [[UV2:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<3 x i32>)
; UNPACKED-NEXT: G_STORE [[UV2]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; UNPACKED-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR1]](i32)
; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
@@ -1523,11 +1523,11 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_1000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
@@ -1590,11 +1590,11 @@ define amdgpu_ps <4 x half> @image_load_tfe_v4f16_dmask_0000(<8 x i32> inreg %rs
; UNPACKED-NEXT: [[AMDGPU_INTRIN_IMAGE_LOAD_D16_:%[0-9]+]]:_(<2 x i32>) = G_AMDGPU_INTRIN_IMAGE_LOAD_D16 intrinsic(@llvm.amdgcn.image.load.2d), 1, [[BUILD_VECTOR1]](<2 x i32>), $noreg, [[BUILD_VECTOR]](<8 x i32>), 1, 0, 0 :: (dereferenceable load (f16), addrspace 8)
; UNPACKED-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AMDGPU_INTRIN_IMAGE_LOAD_D16_]](<2 x i32>)
; UNPACKED-NEXT: G_STORE [[UV1]](i32), [[DEF]](p1) :: (store (i32) into `ptr addrspace(1) poison`, addrspace 1)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; UNPACKED-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNPACKED-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; UNPACKED-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNPACKED-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; UNPACKED-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x f16>) = G_BITCAST [[OR]](i32)
; UNPACKED-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
index 025273821e9f6..25cfa8c7ca8d6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.image.store.2d.d16.ll
@@ -254,18 +254,17 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
; GFX81-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX81-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[COPY11]](<2 x f16>)
; GFX81-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x i32>) = G_BUILD_VECTOR [[COPY8]](i32), [[COPY9]](i32)
- ; GFX81-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX81-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; GFX81-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX81-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; GFX81-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX81-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; GFX81-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; GFX81-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX81-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX81-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; GFX81-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX81-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; GFX81-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C1]]
+ ; GFX81-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX81-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
; GFX81-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX81-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX81-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
+ ; GFX81-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
; GFX81-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX81-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x i16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX81-NEXT: [[BITCAST5:%[0-9]+]]:_(<3 x i32>) = G_BITCAST [[CONCAT_VECTORS]](<6 x i16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
index db19e1f54ccc9..90b19f5422a13 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-llvm.amdgcn.s.buffer.load.mir
@@ -187,30 +187,29 @@ body: |
; GFX67-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[UV2]], [[C1]](i32)
; GFX67-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[UV2]], [[C2]](i32)
; GFX67-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[UV2]], [[C3]](i32)
- ; GFX67-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX67-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C4]]
- ; GFX67-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C4]]
- ; GFX67-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX67-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; GFX67-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX67-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C4]]
+ ; GFX67-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C4]]
+ ; GFX67-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; GFX67-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX67-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX67-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C5]](s32)
+ ; GFX67-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C2]](i32)
; GFX67-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL1]]
; GFX67-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX67-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C4]]
- ; GFX67-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C4]]
- ; GFX67-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; GFX67-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C4]]
+ ; GFX67-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR3]], [[C4]]
+ ; GFX67-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C2]](i32)
; GFX67-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL2]]
; GFX67-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; GFX67-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C5]](s32)
+ ; GFX67-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[LSHR5]], [[C2]](i32)
; GFX67-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL3]]
; GFX67-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; GFX67-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C4]]
- ; GFX67-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C4]]
- ; GFX67-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C5]](s32)
+ ; GFX67-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C4]]
+ ; GFX67-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C4]]
+ ; GFX67-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C2]](i32)
; GFX67-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL4]]
; GFX67-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; GFX67-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[LSHR8]], [[C5]](s32)
+ ; GFX67-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[LSHR8]], [[C2]](i32)
; GFX67-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[LSHR7]], [[SHL5]]
; GFX67-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; GFX67-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<12 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
index fc8a7f8a61397..95b9c4bb3198a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-constant.mir
@@ -3314,10 +3314,9 @@ body: |
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -3345,10 +3344,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -3376,10 +3374,9 @@ body: |
; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -3434,10 +3431,9 @@ body: |
; GFX11-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(p4) = COPY $vgpr0_vgpr1
@@ -3489,10 +3485,9 @@ body: |
; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -3530,10 +3525,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -3571,10 +3565,9 @@ body: |
; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX9-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX9-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -3649,10 +3642,9 @@ body: |
; GFX11-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX11-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX11-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX11-FAKE16-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX11-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX11-FAKE16-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX11-FAKE16-NEXT: $vgpr0 = COPY [[OR4]](s32)
%0:_(p4) = COPY $vgpr0_vgpr1
@@ -4459,11 +4451,11 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (load (s16) from unknown-address + 2, addrspace 4)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -4476,11 +4468,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (load (s16) from unknown-address + 2, addrspace 4)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -4550,11 +4542,11 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load (s8) from unknown-address + 3, addrspace 4)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -4577,11 +4569,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load (s8) from unknown-address + 3, addrspace 4)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -4648,9 +4640,9 @@ body: |
; CI-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
; CI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load (<4 x s16>), addrspace 4)
; CI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -4662,9 +4654,9 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p4) = COPY $vgpr0_vgpr1
; VI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p4) :: (load (<4 x s16>), addrspace 4)
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -4728,19 +4720,19 @@ body: |
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load (s16) from unknown-address + 4, align 4, addrspace 4)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -4757,19 +4749,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load (s16) from unknown-address + 4, align 4, addrspace 4)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -4858,19 +4850,19 @@ body: |
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load (s16) from unknown-address + 4, addrspace 4)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -4887,19 +4879,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p4) :: (load (s16) from unknown-address + 4, addrspace 4)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -5002,19 +4994,19 @@ body: |
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p4) :: (load (s8) from unknown-address + 5, addrspace 4)
; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; CI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -5045,19 +5037,19 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p4) :: (load (s8) from unknown-address + 5, addrspace 4)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -5235,16 +5227,16 @@ body: |
; CI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load (s16) from unknown-address + 6, addrspace 4)
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -5264,16 +5256,16 @@ body: |
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p4) :: (load (s16) from unknown-address + 6, addrspace 4)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -5386,16 +5378,16 @@ body: |
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p4) :: (load (s8) from unknown-address + 7, addrspace 4)
; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; CI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -5433,16 +5425,16 @@ body: |
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p4) :: (load (s8) from unknown-address + 7, addrspace 4)
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
index 7d034129266f0..5c25d3e485f0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-flat.mir
@@ -921,17 +921,18 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i16) from unknown-address + 4, align 4)
; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 281474976710655
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MV]], [[C4]]
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 281474976710655
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MV]], [[C5]]
; CI-NEXT: $vgpr0_vgpr1 = COPY [[AND2]](s64)
;
; VI-LABEL: name: test_load_flat_s48_align8
@@ -944,17 +945,18 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p0) :: (load (i16) from unknown-address + 4, align 4)
; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 281474976710655
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MV]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 281474976710655
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(s64) = G_AND [[MV]], [[C5]]
; VI-NEXT: $vgpr0_vgpr1 = COPY [[AND2]](s64)
;
; GFX9PLUS-LABEL: name: test_load_flat_s48_align8
@@ -4956,10 +4958,9 @@ body: |
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -4987,10 +4988,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -5018,10 +5018,9 @@ body: |
; GFX9PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -5076,10 +5075,9 @@ body: |
; GFX11PLUS-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11PLUS-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11PLUS-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11PLUS-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11PLUS-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11PLUS-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11PLUS-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11PLUS-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11PLUS-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11PLUS-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11PLUS-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11PLUS-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -5134,10 +5132,9 @@ body: |
; GFX12-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX12-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX12-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX12-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -5165,10 +5162,9 @@ body: |
; UNALIGNED_GFX9PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; UNALIGNED_GFX9PLUS-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; UNALIGNED_GFX9PLUS-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; UNALIGNED_GFX9PLUS-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; UNALIGNED_GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; UNALIGNED_GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; UNALIGNED_GFX9PLUS-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -5274,10 +5270,9 @@ body: |
; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -5315,10 +5310,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -5351,10 +5345,9 @@ body: |
; GFX9PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX9PLUS-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX9PLUS-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9PLUS-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX9PLUS-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -5419,10 +5412,9 @@ body: |
; GFX11PLUS-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX11PLUS-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX11PLUS-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX11PLUS-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11PLUS-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11PLUS-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11PLUS-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX11PLUS-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11PLUS-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11PLUS-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11PLUS-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX11PLUS-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -5487,10 +5479,9 @@ body: |
; GFX12-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX12-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX12-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX12-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX12-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -5528,10 +5519,9 @@ body: |
; UNALIGNED_GFX9PLUS-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; UNALIGNED_GFX9PLUS-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; UNALIGNED_GFX9PLUS-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; UNALIGNED_GFX9PLUS-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; UNALIGNED_GFX9PLUS-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; UNALIGNED_GFX9PLUS-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; UNALIGNED_GFX9PLUS-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; UNALIGNED_GFX9PLUS-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -6304,11 +6294,11 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -6321,11 +6311,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -6414,11 +6404,11 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -6441,11 +6431,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -6561,19 +6551,18 @@ body: |
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6589,19 +6578,18 @@ body: |
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6725,19 +6713,18 @@ body: |
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6753,19 +6740,18 @@ body: |
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -6949,19 +6935,19 @@ body: |
; CI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -6978,19 +6964,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -7189,19 +7175,19 @@ body: |
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; CI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -7232,19 +7218,19 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p0) :: (load (s8) from unknown-address + 5)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -7622,11 +7608,11 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
@@ -7634,9 +7620,9 @@ body: |
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C1]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C1]]
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C1]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C1]]
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C2]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -7650,11 +7636,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p0) :: (load (s16) from unknown-address + 2)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
@@ -7662,9 +7648,9 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p0) :: (load (s16) from unknown-address + 4)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i64)
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s16) from unknown-address + 6)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C1]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C1]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C1]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C1]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C2]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -7780,11 +7766,11 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
@@ -7800,9 +7786,9 @@ body: |
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C3]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C3]]
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C3]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C3]]
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -7826,11 +7812,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p0) :: (load (s8) from unknown-address + 3)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
@@ -7846,9 +7832,9 @@ body: |
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p0) :: (load (s8) from unknown-address + 7)
; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C3]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C3]]
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C3]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C3]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
index 19c2bb143eebf..69c4c0aaf2623 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-global.mir
@@ -6280,10 +6280,9 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; SI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6313,10 +6312,9 @@ body: |
; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-HSA-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-HSA-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6346,10 +6344,9 @@ body: |
; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-MESA-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-MESA-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6377,10 +6374,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6408,10 +6404,9 @@ body: |
; GFX9-HSA-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9-HSA-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-HSA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-HSA-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6439,10 +6434,9 @@ body: |
; GFX9-MESA-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9-MESA-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-MESA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-MESA-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6497,10 +6491,9 @@ body: |
; GFX11-HSA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11-HSA-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11-HSA-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-HSA-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-HSA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-HSA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-HSA-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11-HSA-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6555,10 +6548,9 @@ body: |
; GFX11-MESA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11-MESA-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11-MESA-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-MESA-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-MESA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-MESA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-MESA-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11-MESA-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6613,10 +6605,9 @@ body: |
; GFX12-HSA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX12-HSA-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX12-HSA-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX12-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX12-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-HSA-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-HSA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX12-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX12-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-HSA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-HSA-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX12-HSA-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -6671,10 +6662,9 @@ body: |
; GFX12-MESA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX12-MESA-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX12-MESA-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX12-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX12-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-MESA-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-MESA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX12-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX12-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-MESA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-MESA-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX12-MESA-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(p1) = COPY $vgpr0_vgpr1
@@ -6727,10 +6717,9 @@ body: |
; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; SI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -6765,10 +6754,9 @@ body: |
; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
; CI-HSA-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL2]](s32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; CI-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-HSA-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; CI-HSA-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -6808,10 +6796,9 @@ body: |
; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; CI-MESA-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CI-MESA-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CI-MESA-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CI-MESA-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -6849,10 +6836,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -6885,10 +6871,9 @@ body: |
; GFX9-HSA-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX9-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX9-HSA-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX9-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9-HSA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-HSA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX9-HSA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-HSA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9-HSA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-HSA-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX9-HSA-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -6926,10 +6911,9 @@ body: |
; GFX9-MESA-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX9-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX9-MESA-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX9-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX9-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-MESA-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX9-MESA-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9-MESA-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX9-MESA-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX9-MESA-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX9-MESA-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -6994,10 +6978,9 @@ body: |
; GFX11-HSA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX11-HSA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX11-HSA-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX11-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11-HSA-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-HSA-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX11-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11-HSA-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-HSA-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX11-HSA-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -7072,10 +7055,9 @@ body: |
; GFX11-MESA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX11-MESA-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX11-MESA-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX11-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX11-MESA-FAKE16-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-MESA-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX11-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX11-MESA-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX11-MESA-FAKE16-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX11-MESA-FAKE16-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -7140,10 +7122,9 @@ body: |
; GFX12-HSA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX12-HSA-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX12-HSA-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX12-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX12-HSA-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-HSA-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX12-HSA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-HSA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX12-HSA-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-HSA-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX12-HSA-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -7218,10 +7199,9 @@ body: |
; GFX12-MESA-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX12-MESA-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX12-MESA-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX12-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX12-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX12-MESA-FAKE16-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-MESA-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX12-MESA-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX12-MESA-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX12-MESA-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX12-MESA-FAKE16-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX12-MESA-FAKE16-NEXT: $vgpr0 = COPY [[OR4]](s32)
%0:_(p1) = COPY $vgpr0_vgpr1
@@ -8271,11 +8251,11 @@ body: |
; SI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s16) from unknown-address + 2, addrspace 1)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -8295,11 +8275,11 @@ body: |
; CI-MESA-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; CI-MESA-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; CI-MESA-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s16) from unknown-address + 2, addrspace 1)
- ; CI-MESA-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-MESA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -8312,11 +8292,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s16) from unknown-address + 2, addrspace 1)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -8431,11 +8411,11 @@ body: |
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s8) from unknown-address + 3, addrspace 1)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -8465,11 +8445,11 @@ body: |
; CI-MESA-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s8) from unknown-address + 3, addrspace 1)
; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -8492,11 +8472,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s8) from unknown-address + 3, addrspace 1)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -8607,9 +8587,9 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; SI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load (<4 x s16>), addrspace 1)
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -8621,9 +8601,9 @@ body: |
; CI-HSA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; CI-HSA-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load (<4 x s16>), addrspace 1)
; CI-HSA-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; CI-HSA-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-HSA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-HSA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -8635,9 +8615,9 @@ body: |
; CI-MESA-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; CI-MESA-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load (<4 x s16>), addrspace 1)
; CI-MESA-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; CI-MESA-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-MESA-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-MESA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -8649,9 +8629,9 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; VI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p1) :: (load (<4 x s16>), addrspace 1)
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -8798,19 +8778,19 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, align 4, addrspace 1)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -8827,19 +8807,19 @@ body: |
; CI-HSA-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-HSA-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-HSA-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, align 4, addrspace 1)
- ; CI-HSA-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -8856,19 +8836,19 @@ body: |
; CI-MESA-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-MESA-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-MESA-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, align 4, addrspace 1)
- ; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-MESA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -8885,19 +8865,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, align 4, addrspace 1)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9124,19 +9104,19 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, addrspace 1)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9153,19 +9133,19 @@ body: |
; CI-HSA-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-HSA-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-HSA-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, addrspace 1)
- ; CI-HSA-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9182,19 +9162,19 @@ body: |
; CI-MESA-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-MESA-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-MESA-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, addrspace 1)
- ; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-MESA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9211,19 +9191,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, addrspace 1)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9464,19 +9444,19 @@ body: |
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p1) :: (load (s8) from unknown-address + 5, addrspace 1)
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9493,19 +9473,19 @@ body: |
; CI-HSA-NEXT: [[C1:%[0-9]+]]:_(i64) = G_CONSTANT i64 4
; CI-HSA-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i64)
; CI-HSA-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s16) from unknown-address + 4, align 1, addrspace 1)
- ; CI-HSA-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9536,19 +9516,19 @@ body: |
; CI-MESA-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p1) :: (load (s8) from unknown-address + 5, addrspace 1)
; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-MESA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-MESA-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-MESA-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-MESA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9579,19 +9559,19 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p1) :: (load (s8) from unknown-address + 5, addrspace 1)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -9987,16 +9967,16 @@ body: |
; SI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s16) from unknown-address + 6, addrspace 1)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -10023,16 +10003,16 @@ body: |
; CI-MESA-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; CI-MESA-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; CI-MESA-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s16) from unknown-address + 6, addrspace 1)
- ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CI-MESA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -10052,16 +10032,16 @@ body: |
; VI-NEXT: [[C2:%[0-9]+]]:_(i64) = G_CONSTANT i64 6
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i64)
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s16) from unknown-address + 6, addrspace 1)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -10237,16 +10217,16 @@ body: |
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p1) :: (load (s8) from unknown-address + 7, addrspace 1)
; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -10291,16 +10271,16 @@ body: |
; CI-MESA-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p1) :: (load (s8) from unknown-address + 7, addrspace 1)
; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; CI-MESA-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; CI-MESA-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; CI-MESA-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-MESA-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -10338,16 +10318,16 @@ body: |
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p1) :: (load (s8) from unknown-address + 7, addrspace 1)
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -10714,11 +10694,11 @@ body: |
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s16) from unknown-address + 8, align 8, addrspace 1)
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C3]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[UV]](<2 x s16>)
@@ -10742,21 +10722,21 @@ body: |
; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-HSA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-HSA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 8, addrspace 1)
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10780,21 +10760,21 @@ body: |
; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-MESA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-MESA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 8, addrspace 1)
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10818,21 +10798,21 @@ body: |
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 8, addrspace 1)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11144,11 +11124,11 @@ body: |
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i64)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s16) from unknown-address + 8, align 4, addrspace 1)
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C3]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[UV]](<2 x s16>)
@@ -11172,21 +11152,21 @@ body: |
; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-HSA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-HSA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 4, addrspace 1)
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11210,21 +11190,21 @@ body: |
; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-MESA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-MESA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 4, addrspace 1)
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11248,21 +11228,21 @@ body: |
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 4, addrspace 1)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11582,21 +11562,21 @@ body: |
; SI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; SI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; SI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, addrspace 1)
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11620,21 +11600,21 @@ body: |
; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-HSA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-HSA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, addrspace 1)
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11658,21 +11638,21 @@ body: |
; CI-MESA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-MESA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-MESA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, addrspace 1)
- ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -11696,21 +11676,21 @@ body: |
; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; VI-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, addrspace 1)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -12052,21 +12032,21 @@ body: |
; SI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p1) :: (load (s8) from unknown-address + 9, addrspace 1)
; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C6]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C6]]
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C6]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C6]]
+ ; SI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL5]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C6]]
- ; SI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C6]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C6]]
+ ; SI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; SI-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL6]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C6]]
- ; SI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C6]]
+ ; SI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; SI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL7]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -12090,21 +12070,21 @@ body: |
; CI-HSA-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 8
; CI-HSA-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; CI-HSA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s16) from unknown-address + 8, align 1, addrspace 1)
- ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C4]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C4]]
- ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C4]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C4]]
+ ; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C4]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C4]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C4]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C4]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C5]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C4]]
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C4]]
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -12150,21 +12130,21 @@ body: |
; CI-MESA-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p1) :: (load (s8) from unknown-address + 9, addrspace 1)
; CI-MESA-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
; CI-MESA-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C6]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C6]]
- ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C6]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C6]]
+ ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-MESA-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL5]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C6]]
- ; CI-MESA-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C6]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-MESA-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL6]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C6]]
- ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C6]]
+ ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-MESA-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL7]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -12210,21 +12190,21 @@ body: |
; VI-NEXT: [[LOAD4:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD8]](p1) :: (load (s8) from unknown-address + 9, addrspace 1)
; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD4]], [[C1]](s32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C6]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C6]]
- ; VI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C6]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C6]]
+ ; VI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL5]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C6]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C6]]
- ; VI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C6]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C6]]
+ ; VI-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; VI-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL6]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C6]]
- ; VI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C6]]
+ ; VI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; VI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL7]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -13659,26 +13639,26 @@ body: |
; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; SI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; SI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; SI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; SI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; SI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -13709,26 +13689,26 @@ body: |
; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-HSA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-HSA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-HSA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-HSA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -13759,26 +13739,26 @@ body: |
; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-MESA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-MESA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-MESA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-MESA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -13809,26 +13789,26 @@ body: |
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; VI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; VI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; VI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; VI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; VI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; VI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; VI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; VI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14248,26 +14228,26 @@ body: |
; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; SI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; SI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; SI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; SI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; SI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14298,26 +14278,26 @@ body: |
; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-HSA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-HSA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-HSA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-HSA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14348,26 +14328,26 @@ body: |
; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-MESA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-MESA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-MESA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-MESA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14398,26 +14378,26 @@ body: |
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 4, addrspace 1)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; VI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; VI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; VI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; VI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; VI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; VI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; VI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; VI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14837,26 +14817,26 @@ body: |
; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; SI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; SI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, addrspace 1)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; SI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; SI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; SI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14887,26 +14867,26 @@ body: |
; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-HSA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-HSA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, addrspace 1)
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-HSA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-HSA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14937,26 +14917,26 @@ body: |
; CI-MESA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-MESA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-MESA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, addrspace 1)
- ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-MESA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-MESA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-MESA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-MESA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-MESA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-MESA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-MESA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -14987,26 +14967,26 @@ body: |
; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; VI-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, addrspace 1)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; VI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; VI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; VI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; VI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; VI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; VI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; VI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; VI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -15456,26 +15436,26 @@ body: |
; SI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p1) :: (load (s8) from unknown-address + 13, addrspace 1)
; SI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LOAD6]], [[C1]](s32)
; SI-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
- ; SI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C8]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C8]]
- ; SI-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C9]](s32)
+ ; SI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C8]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C8]]
+ ; SI-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C9]](i32)
; SI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL7]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C8]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C8]]
- ; SI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C9]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C8]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C8]]
+ ; SI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C9]](i32)
; SI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C8]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[OR5]], [[C8]]
- ; SI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C9]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C8]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[OR5]], [[C8]]
+ ; SI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C9]](i32)
; SI-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL9]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[OR6]], [[C8]]
- ; SI-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C10]], [[C9]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[OR6]], [[C8]]
+ ; SI-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C10]], [[C9]](i32)
; SI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL10]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -15506,26 +15486,26 @@ body: |
; CI-HSA-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 12
; CI-HSA-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; CI-HSA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD5]](p1) :: (load (s16) from unknown-address + 12, align 1, addrspace 1)
- ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C6]]
- ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C6]]
- ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-HSA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C6]]
+ ; CI-HSA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C6]]
+ ; CI-HSA-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-HSA-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C7]](i32)
; CI-HSA-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-HSA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C6]]
- ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C6]]
- ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C6]]
+ ; CI-HSA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; CI-HSA-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-HSA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LOAD4]], [[C6]]
- ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LOAD5]], [[C6]]
- ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LOAD4]], [[C6]]
+ ; CI-HSA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LOAD5]], [[C6]]
+ ; CI-HSA-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C7]](i32)
; CI-HSA-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; CI-HSA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LOAD6]], [[C6]]
- ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C8]], [[C7]](s32)
+ ; CI-HSA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LOAD6]], [[C6]]
+ ; CI-HSA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-HSA-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C8]], [[C7]](i32)
; CI-HSA-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL3]]
; CI-HSA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-HSA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -15586,26 +15566,26 @@ body: |
; CI-MESA-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p1) :: (load (s8) from unknown-address + 13, addrspace 1)
; CI-MESA-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LOAD6]], [[C1]](s32)
; CI-MESA-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
- ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C8]]
- ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C8]]
- ; CI-MESA-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-MESA-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C9]](s32)
+ ; CI-MESA-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-MESA-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C8]]
+ ; CI-MESA-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C8]]
+ ; CI-MESA-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-MESA-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C9]](i32)
; CI-MESA-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL7]]
; CI-MESA-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
- ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C8]]
- ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C8]]
- ; CI-MESA-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C9]](s32)
+ ; CI-MESA-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C8]]
+ ; CI-MESA-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C8]]
+ ; CI-MESA-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C9]](i32)
; CI-MESA-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; CI-MESA-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C8]]
- ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[OR5]], [[C8]]
- ; CI-MESA-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C9]](s32)
+ ; CI-MESA-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C8]]
+ ; CI-MESA-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[OR5]], [[C8]]
+ ; CI-MESA-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C9]](i32)
; CI-MESA-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL9]]
; CI-MESA-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[OR6]], [[C8]]
- ; CI-MESA-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-MESA-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C10]], [[C9]](s32)
+ ; CI-MESA-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[OR6]], [[C8]]
+ ; CI-MESA-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-MESA-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C10]], [[C9]](i32)
; CI-MESA-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL10]]
; CI-MESA-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
; CI-MESA-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -15666,26 +15646,26 @@ body: |
; VI-NEXT: [[LOAD6:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD12]](p1) :: (load (s8) from unknown-address + 13, addrspace 1)
; VI-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LOAD6]], [[C1]](s32)
; VI-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[SHL6]], [[ZEXTLOAD6]]
- ; VI-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C8]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C8]]
- ; VI-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C9]](s32)
+ ; VI-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C8]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C8]]
+ ; VI-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C9]](i32)
; VI-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL7]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C8]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C8]]
- ; VI-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C9]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C8]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C8]]
+ ; VI-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C9]](i32)
; VI-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C8]]
- ; VI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[OR5]], [[C8]]
- ; VI-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C9]](s32)
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C8]]
+ ; VI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[OR5]], [[C8]]
+ ; VI-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C9]](i32)
; VI-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL9]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; VI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[OR6]], [[C8]]
- ; VI-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C10]], [[C9]](s32)
+ ; VI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[OR6]], [[C8]]
+ ; VI-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C10]], [[C9]](i32)
; VI-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL10]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
index 41cdf2beb3b60..49cafa4f7daca 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-local.mir
@@ -8423,10 +8423,9 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; SI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8456,10 +8455,9 @@ body: |
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8489,10 +8487,9 @@ body: |
; CI-DS128-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-DS128-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-DS128-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-DS128-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-DS128-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-DS128-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-DS128-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-DS128-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-DS128-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-DS128-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8520,10 +8517,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8551,10 +8547,9 @@ body: |
; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8582,10 +8577,9 @@ body: |
; GFX9-UNALIGNED-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9-UNALIGNED-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9-UNALIGNED-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-UNALIGNED-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-UNALIGNED-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-UNALIGNED-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-UNALIGNED-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-UNALIGNED-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8613,10 +8607,9 @@ body: |
; GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX10-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8644,10 +8637,9 @@ body: |
; GFX10-UNALIGNED-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX10-UNALIGNED-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX10-UNALIGNED-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX10-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX10-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX10-UNALIGNED-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-UNALIGNED-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX10-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX10-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX10-UNALIGNED-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX10-UNALIGNED-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX10-UNALIGNED-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8702,10 +8694,9 @@ body: |
; GFX11-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8760,10 +8751,9 @@ body: |
; GFX11-UNALIGNED-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11-UNALIGNED-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11-UNALIGNED-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-UNALIGNED-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-UNALIGNED-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11-UNALIGNED-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8818,10 +8808,9 @@ body: |
; GFX12-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX12-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX12-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX12-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8876,10 +8865,9 @@ body: |
; GFX12-UNALIGNED-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX12-UNALIGNED-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX12-UNALIGNED-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-UNALIGNED-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-UNALIGNED-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX12-UNALIGNED-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(p3) = COPY $vgpr0
@@ -8931,10 +8919,9 @@ body: |
; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; SI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -8974,10 +8961,9 @@ body: |
; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9017,10 +9003,9 @@ body: |
; CI-DS128-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; CI-DS128-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CI-DS128-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-DS128-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-DS128-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CI-DS128-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; CI-DS128-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-DS128-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CI-DS128-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CI-DS128-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CI-DS128-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9058,10 +9043,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9099,10 +9083,9 @@ body: |
; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX9-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX9-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9135,10 +9118,9 @@ body: |
; GFX9-UNALIGNED-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX9-UNALIGNED-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX9-UNALIGNED-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX9-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9-UNALIGNED-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-UNALIGNED-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX9-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9-UNALIGNED-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-UNALIGNED-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX9-UNALIGNED-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -9176,10 +9158,9 @@ body: |
; GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX10-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX10-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9212,10 +9193,9 @@ body: |
; GFX10-UNALIGNED-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX10-UNALIGNED-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX10-UNALIGNED-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX10-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX10-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX10-UNALIGNED-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-UNALIGNED-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX10-UNALIGNED-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX10-UNALIGNED-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX10-UNALIGNED-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX10-UNALIGNED-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX10-UNALIGNED-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -9290,10 +9270,9 @@ body: |
; GFX11-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX11-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX11-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX11-FAKE16-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX11-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX11-FAKE16-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX11-FAKE16-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9358,10 +9337,9 @@ body: |
; GFX11-UNALIGNED-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX11-UNALIGNED-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX11-UNALIGNED-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-UNALIGNED-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11-UNALIGNED-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-UNALIGNED-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX11-UNALIGNED-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -9436,10 +9414,9 @@ body: |
; GFX12-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; GFX12-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; GFX12-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; GFX12-FAKE16-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; GFX12-FAKE16-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX12-FAKE16-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; GFX12-FAKE16-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9504,10 +9481,9 @@ body: |
; GFX12-UNALIGNED-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX12-UNALIGNED-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX12-UNALIGNED-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-UNALIGNED-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-UNALIGNED-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX12-UNALIGNED-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-UNALIGNED-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX12-UNALIGNED-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
%0:_(p3) = COPY $vgpr0
@@ -10406,11 +10382,11 @@ body: |
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s16) from unknown-address + 2, addrspace 3)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10423,11 +10399,11 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s16) from unknown-address + 2, addrspace 3)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10440,11 +10416,11 @@ body: |
; CI-DS128-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; CI-DS128-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; CI-DS128-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s16) from unknown-address + 2, addrspace 3)
- ; CI-DS128-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-DS128-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-DS128-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-DS128-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-DS128-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-DS128-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10457,11 +10433,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p3) :: (load (s16) from unknown-address + 2, addrspace 3)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10596,11 +10572,11 @@ body: |
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s8) from unknown-address + 3, addrspace 3)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10623,11 +10599,11 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s8) from unknown-address + 3, addrspace 3)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10650,11 +10626,11 @@ body: |
; CI-DS128-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s8) from unknown-address + 3, addrspace 3)
; CI-DS128-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-DS128-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-DS128-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-DS128-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-DS128-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-DS128-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10677,11 +10653,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s8) from unknown-address + 3, addrspace 3)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -10822,9 +10798,9 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; SI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load (<4 x s16>), addrspace 3)
; SI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -10836,9 +10812,9 @@ body: |
; CI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; CI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load (<4 x s16>), addrspace 3)
; CI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -10850,9 +10826,9 @@ body: |
; CI-DS128-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; CI-DS128-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load (<4 x s16>), addrspace 3)
; CI-DS128-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; CI-DS128-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-DS128-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-DS128-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-DS128-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-DS128-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-DS128-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -10864,9 +10840,9 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p3) = COPY $vgpr0
; VI-NEXT: [[LOAD:%[0-9]+]]:_(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load (<4 x s16>), addrspace 3)
; VI-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[LOAD]](<4 x s16>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[UV1]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -11035,19 +11011,19 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s16) from unknown-address + 4, addrspace 3)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11064,19 +11040,19 @@ body: |
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s16) from unknown-address + 4, addrspace 3)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11093,19 +11069,19 @@ body: |
; CI-DS128-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-DS128-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; CI-DS128-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s16) from unknown-address + 4, addrspace 3)
- ; CI-DS128-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-DS128-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-DS128-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-DS128-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-DS128-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-DS128-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-DS128-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-DS128-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-DS128-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-DS128-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-DS128-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-DS128-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11122,19 +11098,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p3) :: (load (s16) from unknown-address + 4, addrspace 3)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11417,19 +11393,19 @@ body: |
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load (s8) from unknown-address + 5, addrspace 3)
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11460,19 +11436,19 @@ body: |
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load (s8) from unknown-address + 5, addrspace 3)
; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; CI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11503,19 +11479,19 @@ body: |
; CI-DS128-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load (s8) from unknown-address + 5, addrspace 3)
; CI-DS128-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; CI-DS128-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; CI-DS128-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; CI-DS128-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-DS128-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; CI-DS128-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-DS128-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; CI-DS128-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-DS128-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-DS128-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CI-DS128-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-DS128-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-DS128-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-DS128-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; CI-DS128-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-DS128-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11546,19 +11522,19 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p3) :: (load (s8) from unknown-address + 5, addrspace 3)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -11943,16 +11919,16 @@ body: |
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s16) from unknown-address + 6, addrspace 3)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12090,16 +12066,16 @@ body: |
; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s16) from unknown-address + 6, addrspace 3)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12119,16 +12095,16 @@ body: |
; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s16) from unknown-address + 6, addrspace 3)
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12148,16 +12124,16 @@ body: |
; CI-DS128-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
; CI-DS128-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; CI-DS128-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s16) from unknown-address + 6, addrspace 3)
- ; CI-DS128-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-DS128-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; CI-DS128-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-DS128-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; CI-DS128-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; CI-DS128-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; CI-DS128-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; CI-DS128-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; CI-DS128-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-DS128-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CI-DS128-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12177,16 +12153,16 @@ body: |
; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p3) = nuw inbounds G_PTR_ADD [[COPY]], [[C2]](i32)
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p3) :: (load (s16) from unknown-address + 6, addrspace 3)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C3]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C3]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C3]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C3]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12408,16 +12384,16 @@ body: |
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load (s8) from unknown-address + 7, addrspace 3)
; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12455,16 +12431,16 @@ body: |
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load (s8) from unknown-address + 7, addrspace 3)
; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; CI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12502,16 +12478,16 @@ body: |
; CI-DS128-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load (s8) from unknown-address + 7, addrspace 3)
; CI-DS128-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; CI-DS128-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; CI-DS128-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; CI-DS128-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-DS128-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; CI-DS128-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-DS128-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; CI-DS128-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; CI-DS128-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-DS128-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; CI-DS128-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; CI-DS128-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; CI-DS128-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; CI-DS128-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; CI-DS128-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; CI-DS128-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; CI-DS128-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; CI-DS128-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; CI-DS128-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-DS128-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -12549,16 +12525,16 @@ body: |
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p3) :: (load (s8) from unknown-address + 7, addrspace 3)
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[SHL3]], [[ZEXTLOAD3]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C5]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C5]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C5]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C5]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C5]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C5]]
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C6]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C5]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
index 49e87c3b90835..b756917a61571 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-load-private.mir
@@ -1727,13 +1727,14 @@ body: |
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; SI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; SI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; SI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1748,13 +1749,14 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; CI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1769,13 +1771,14 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; VI-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; VI-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1790,13 +1793,14 @@ body: |
; GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1811,13 +1815,14 @@ body: |
; GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX10-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1846,13 +1851,14 @@ body: |
; UNALIGNED_GFX9-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; UNALIGNED_GFX9-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; UNALIGNED_GFX9-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNALIGNED_GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; UNALIGNED_GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNALIGNED_GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNALIGNED_GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; UNALIGNED_GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; UNALIGNED_GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; UNALIGNED_GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; UNALIGNED_GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNALIGNED_GFX9-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX9-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -1867,13 +1873,14 @@ body: |
; UNALIGNED_GFX10-NEXT: [[LOAD1:%[0-9]+]]:_(i32) = G_LOAD [[PTR_ADD]](p5) :: (load (i16) from unknown-address + 4, align 4, addrspace 5)
; UNALIGNED_GFX10-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; UNALIGNED_GFX10-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[LOAD]], [[C1]](s32)
- ; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; UNALIGNED_GFX10-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; UNALIGNED_GFX10-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; UNALIGNED_GFX10-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; UNALIGNED_GFX10-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C3]](i32)
; UNALIGNED_GFX10-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; UNALIGNED_GFX10-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; UNALIGNED_GFX10-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; UNALIGNED_GFX10-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; UNALIGNED_GFX10-NEXT: [[MV:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
; UNALIGNED_GFX10-NEXT: $vgpr0_vgpr1 = COPY [[MV]](i64)
@@ -7755,10 +7762,9 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; SI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -7788,10 +7794,9 @@ body: |
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -7819,10 +7824,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -7850,10 +7854,9 @@ body: |
; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -7881,10 +7884,9 @@ body: |
; GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX10-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX10-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -7939,10 +7941,9 @@ body: |
; GFX11-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX11-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX11-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX11-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -7997,10 +7998,9 @@ body: |
; GFX12-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; GFX12-FAKE16-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; GFX12-FAKE16-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-FAKE16-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX12-FAKE16-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8028,10 +8028,9 @@ body: |
; UNALIGNED_GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; UNALIGNED_GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; UNALIGNED_GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; UNALIGNED_GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; UNALIGNED_GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; UNALIGNED_GFX9-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; UNALIGNED_GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; UNALIGNED_GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; UNALIGNED_GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; UNALIGNED_GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8059,10 +8058,9 @@ body: |
; UNALIGNED_GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C2]]
; UNALIGNED_GFX10-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C3]](s16)
; UNALIGNED_GFX10-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; UNALIGNED_GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; UNALIGNED_GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; UNALIGNED_GFX10-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; UNALIGNED_GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; UNALIGNED_GFX10-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; UNALIGNED_GFX10-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; UNALIGNED_GFX10-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -8168,10 +8166,9 @@ body: |
; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; SI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; SI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -8211,10 +8208,9 @@ body: |
; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
; CI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CI-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C7]](s32)
+ ; CI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; CI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -8252,10 +8248,9 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -8288,10 +8283,9 @@ body: |
; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX9-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX9-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX9-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -8324,10 +8318,9 @@ body: |
; GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX10-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX10-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX10-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX10-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX10-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX10-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -8392,10 +8385,9 @@ body: |
; GFX11-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX11-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX11-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX11-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX11-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX11-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX11-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX11-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX11-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX11-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -8460,10 +8452,9 @@ body: |
; GFX12-FAKE16-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C3]]
; GFX12-FAKE16-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C4]](s16)
; GFX12-FAKE16-NEXT: [[OR2:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL2]]
- ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; GFX12-FAKE16-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX12-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; GFX12-FAKE16-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX12-FAKE16-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; GFX12-FAKE16-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C1]](i32)
; GFX12-FAKE16-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL3]]
; GFX12-FAKE16-NEXT: $vgpr0 = COPY [[OR3]](s32)
;
@@ -8501,10 +8492,9 @@ body: |
; UNALIGNED_GFX9-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; UNALIGNED_GFX9-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; UNALIGNED_GFX9-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; UNALIGNED_GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; UNALIGNED_GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; UNALIGNED_GFX9-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX9-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; UNALIGNED_GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; UNALIGNED_GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; UNALIGNED_GFX9-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; UNALIGNED_GFX9-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; UNALIGNED_GFX9-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -8542,10 +8532,9 @@ body: |
; UNALIGNED_GFX10-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C4]]
; UNALIGNED_GFX10-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C5]](s16)
; UNALIGNED_GFX10-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL3]]
- ; UNALIGNED_GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; UNALIGNED_GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; UNALIGNED_GFX10-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; UNALIGNED_GFX10-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C6]](s32)
+ ; UNALIGNED_GFX10-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; UNALIGNED_GFX10-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; UNALIGNED_GFX10-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; UNALIGNED_GFX10-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
; UNALIGNED_GFX10-NEXT: $vgpr0 = COPY [[OR4]](s32)
;
@@ -9469,11 +9458,11 @@ body: |
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -9486,11 +9475,11 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -9503,11 +9492,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -9616,11 +9605,11 @@ body: |
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -9643,11 +9632,11 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -9670,11 +9659,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -9820,19 +9809,18 @@ body: |
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; SI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -9848,19 +9836,18 @@ body: |
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -9876,19 +9863,18 @@ body: |
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(i32) = G_BITCAST [[LOAD]](<2 x s16>)
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C2]]
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -10074,19 +10060,19 @@ body: |
; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -10103,19 +10089,19 @@ body: |
; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; CI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -10132,19 +10118,19 @@ body: |
; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C1]](i32)
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C2]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C2]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C2]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C2]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C3]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C2]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C3]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -10385,19 +10371,19 @@ body: |
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; SI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; SI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -10428,19 +10414,19 @@ body: |
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; CI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; CI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; CI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -10471,19 +10457,19 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD4]](p5) :: (load (s8) from unknown-address + 5, addrspace 5)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LOAD2]], [[C1]](s32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[SHL2]], [[ZEXTLOAD2]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C4]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C4]]
- ; VI-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C4]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C4]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL3]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR2]], [[C4]]
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR2]], [[C4]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL5]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -10997,11 +10983,11 @@ body: |
; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
@@ -11009,9 +10995,9 @@ body: |
; SI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C1]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C1]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C2]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -11025,11 +11011,11 @@ body: |
; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; CI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; CI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; CI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
@@ -11037,9 +11023,9 @@ body: |
; CI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; CI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C1]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C1]]
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C1]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C1]]
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C2]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -11053,11 +11039,11 @@ body: |
; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[COPY]], [[C]](i32)
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p5) :: (load (s16) from unknown-address + 2, addrspace 5)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LOAD]], [[C1]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LOAD1]], [[C1]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LOAD]], [[C1]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LOAD1]], [[C1]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
@@ -11065,9 +11051,9 @@ body: |
; VI-NEXT: [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p5) :: (load (s16) from unknown-address + 4, addrspace 5)
; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p5) = nuw inbounds G_PTR_ADD [[PTR_ADD1]], [[C]](i32)
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s16) from unknown-address + 6, addrspace 5)
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LOAD2]], [[C1]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LOAD3]], [[C1]]
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LOAD2]], [[C1]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LOAD3]], [[C1]]
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C2]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -11219,11 +11205,11 @@ body: |
; SI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; SI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
@@ -11239,9 +11225,9 @@ body: |
; SI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; SI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C3]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C3]]
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C3]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C3]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; SI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -11265,11 +11251,11 @@ body: |
; CI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; CI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; CI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; CI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; CI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; CI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; CI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
@@ -11285,9 +11271,9 @@ body: |
; CI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
; CI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; CI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
- ; CI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C3]]
- ; CI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C3]]
- ; CI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; CI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C3]]
+ ; CI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C3]]
+ ; CI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; CI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -11311,11 +11297,11 @@ body: |
; VI-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p5) :: (load (s8) from unknown-address + 3, addrspace 5)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LOAD1]], [[C1]](s32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[SHL1]], [[ZEXTLOAD1]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OR]], [[C3]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[OR1]], [[C3]]
- ; VI-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C4]](s32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[OR]], [[C3]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[OR1]], [[C3]]
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
@@ -11331,9 +11317,9 @@ body: |
; VI-NEXT: [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD6]](p5) :: (load (s8) from unknown-address + 7, addrspace 5)
; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LOAD3]], [[C1]](s32)
; VI-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[SHL4]], [[ZEXTLOAD3]]
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[OR3]], [[C3]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[OR4]], [[C3]]
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[OR3]], [[C3]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[OR4]], [[C3]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C4]](i32)
; VI-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL5]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
index 883b256c61bb0..904a595596e01 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-lshr.mir
@@ -570,10 +570,10 @@ body: |
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[AND1]], [[AND]](s32)
; SI-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[LSHR1]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR3]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -594,10 +594,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -634,10 +633,10 @@ body: |
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
; SI-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[AND]], [[UV]](s32)
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[UV1]](s32)
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -657,10 +656,9 @@ body: |
; VI-NEXT: [[LSHR1:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR1]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR1]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR2]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -716,15 +714,15 @@ body: |
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LSHR3]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -753,15 +751,14 @@ body: |
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[TRUNC5]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR4]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -829,18 +826,18 @@ body: |
; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[LSHR4:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR2]], [[C1]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR3]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LSHR2]], [[C2]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LSHR3]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
@@ -869,19 +866,18 @@ body: |
; VI-NEXT: [[LSHR2:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[LSHR4:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[TRUNC5]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR2]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR3]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR2]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR3]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR4]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -953,15 +949,15 @@ body: |
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[LSHR6:%[0-9]+]]:_(s32) = G_LSHR [[AND3]], [[AND2]](s32)
; SI-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[LSHR1]], [[LSHR3]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C1]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[LSHR5]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[LSHR4]], [[C2]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[LSHR5]], [[C2]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C1]]
- ; SI-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C2]](s32)
+ ; SI-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C2]]
+ ; SI-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[LSHR7]], [[C2]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -995,15 +991,14 @@ body: |
; VI-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC1]], [[TRUNC5]](s16)
; VI-NEXT: [[LSHR6:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC2]], [[TRUNC6]](s16)
; VI-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[TRUNC3]], [[TRUNC7]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR4]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR5]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR4]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR5]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR6]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[LSHR7]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR6]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[LSHR7]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir
index b56a2b03c1d6c..7c38263b2d9fd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-merge-values.mir
@@ -53,14 +53,14 @@ body: |
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
; CHECK-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[AND6]], [[TRUNC7]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C3]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C3]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](p1)
@@ -124,10 +124,10 @@ body: |
; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[C3]], [[TRUNC1]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C5]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C5]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; CHECK-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(s8) = G_CONSTANT i8 0
@@ -147,14 +147,14 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C4]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C5]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C6]](s32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C4]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C5]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C6]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; CHECK-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(s8) = G_CONSTANT i8 0
@@ -195,15 +195,15 @@ body: |
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[COPY3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[MV]](s64)
@@ -261,10 +261,10 @@ body: |
; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C8]](s32)
; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL5]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s16) = G_OR [[OR4]], [[TRUNC5]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
- ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C10]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR5]](s16)
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C10]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL6]]
; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s24) = G_TRUNC [[OR6]](s32)
; CHECK-NEXT: S_NOP 0, implicit [[TRUNC6]](s24)
@@ -319,10 +319,10 @@ body: |
; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C9]](s32)
; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL5]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s16) = G_OR [[OR4]], [[TRUNC5]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
- ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C11]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR5]](s16)
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C11]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL6]]
; CHECK-NEXT: [[TRUNC6:%[0-9]+]]:_(s28) = G_TRUNC [[OR6]](s32)
; CHECK-NEXT: S_NOP 0, implicit [[TRUNC6]](s28)
@@ -350,25 +350,26 @@ body: |
; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C4]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C8]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C9]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C10]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
- ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C11]](s32)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C12]](s32)
- ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C13]](s32)
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C8]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C9]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C10]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C11]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; CHECK-NEXT: [[C12:%[0-9]+]]:_(i32) = G_CONSTANT i32 20
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C5]], [[C12]](i32)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; CHECK-NEXT: [[C13:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C13]](i32)
+ ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; CHECK-NEXT: [[C14:%[0-9]+]]:_(i32) = G_CONSTANT i32 28
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C7]], [[C14]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
; CHECK-NEXT: S_NOP 0, implicit [[OR6]](s32)
%0:_(s4) = G_CONSTANT i4 0
@@ -412,14 +413,14 @@ body: |
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C5]](s32)
; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[C8]], [[TRUNC3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C9]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C9]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C9]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C9]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
@@ -478,18 +479,18 @@ body: |
; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[COPY6]](s32)
; CHECK-NEXT: [[TRUNC5:%[0-9]+]]:_(s16) = G_TRUNC [[SHL5]](s32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s16) = G_OR [[C12]], [[TRUNC5]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C13]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C13:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C13]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL6]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C13]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C13]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL7]]
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR4]](s16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR5]](s16)
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C13]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR4]](s16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR5]](s16)
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C13]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[ZEXT4]], [[SHL8]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[OR6]](s32), [[OR7]](s32), [[OR8]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[MV]](s96)
@@ -515,18 +516,18 @@ name: test_merge_s96_s16_s16_s16_s16_s16_s16
body: |
bb.0:
; CHECK-LABEL: name: test_merge_s96_s16_s16_s16_s16_s16_s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C2]](s32)
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 3
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C2]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[C3]], [[SHL1]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C2]](s32)
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 5
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C2]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[C5]], [[SHL2]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32), [[OR2]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[MV]](s96)
@@ -571,14 +572,14 @@ body: |
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C4]](s32)
; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL3]](s32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s16) = G_OR [[C7]], [[TRUNC3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C9]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C9]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](s16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](s16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C9]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](s16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](s16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C9]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](s32), [[OR5]](s32)
; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s56) = G_TRUNC [[MV]](s64)
@@ -601,223 +602,225 @@ body: |
; CHECK-LABEL: name: test_merge_s68_s17_s17_s17_s17
; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 3
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C3]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C4]](s32)
- ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(s32) = G_OR [[OR2]], [[SHL3]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C5]](s32)
- ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(s32) = G_OR [[OR3]], [[SHL4]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C6]](s32)
- ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(s32) = G_OR [[OR4]], [[SHL5]]
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 7
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C7]](s32)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(s32) = G_OR [[OR5]], [[SHL6]]
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C8]](s32)
- ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(s32) = G_OR [[OR6]], [[SHL7]]
- ; CHECK-NEXT: [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 9
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C9]](s32)
- ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(s32) = G_OR [[OR7]], [[SHL8]]
- ; CHECK-NEXT: [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C10]](s32)
- ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(s32) = G_OR [[OR8]], [[SHL9]]
- ; CHECK-NEXT: [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 11
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C11]](s32)
- ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(s32) = G_OR [[OR9]], [[SHL10]]
- ; CHECK-NEXT: [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 12
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C12]](s32)
- ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(s32) = G_OR [[OR10]], [[SHL11]]
- ; CHECK-NEXT: [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 13
- ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C13]](s32)
- ; CHECK-NEXT: [[OR12:%[0-9]+]]:_(s32) = G_OR [[OR11]], [[SHL12]]
- ; CHECK-NEXT: [[C14:%[0-9]+]]:_(s32) = G_CONSTANT i32 14
- ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C14]](s32)
- ; CHECK-NEXT: [[OR13:%[0-9]+]]:_(s32) = G_OR [[OR12]], [[SHL13]]
- ; CHECK-NEXT: [[C15:%[0-9]+]]:_(s32) = G_CONSTANT i32 15
- ; CHECK-NEXT: [[SHL14:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C15]](s32)
- ; CHECK-NEXT: [[OR14:%[0-9]+]]:_(s32) = G_OR [[OR13]], [[SHL14]]
- ; CHECK-NEXT: [[C16:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL15:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C16]](s32)
- ; CHECK-NEXT: [[OR15:%[0-9]+]]:_(s32) = G_OR [[OR14]], [[SHL15]]
- ; CHECK-NEXT: [[C17:%[0-9]+]]:_(s32) = G_CONSTANT i32 17
- ; CHECK-NEXT: [[SHL16:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C17]](s32)
- ; CHECK-NEXT: [[OR16:%[0-9]+]]:_(s32) = G_OR [[OR15]], [[SHL16]]
- ; CHECK-NEXT: [[C18:%[0-9]+]]:_(s32) = G_CONSTANT i32 18
- ; CHECK-NEXT: [[SHL17:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C18]](s32)
- ; CHECK-NEXT: [[OR17:%[0-9]+]]:_(s32) = G_OR [[OR16]], [[SHL17]]
- ; CHECK-NEXT: [[C19:%[0-9]+]]:_(s32) = G_CONSTANT i32 19
- ; CHECK-NEXT: [[SHL18:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C19]](s32)
- ; CHECK-NEXT: [[OR18:%[0-9]+]]:_(s32) = G_OR [[OR17]], [[SHL18]]
- ; CHECK-NEXT: [[C20:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
- ; CHECK-NEXT: [[SHL19:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C20]](s32)
- ; CHECK-NEXT: [[OR19:%[0-9]+]]:_(s32) = G_OR [[OR18]], [[SHL19]]
- ; CHECK-NEXT: [[C21:%[0-9]+]]:_(s32) = G_CONSTANT i32 21
- ; CHECK-NEXT: [[SHL20:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C21]](s32)
- ; CHECK-NEXT: [[OR20:%[0-9]+]]:_(s32) = G_OR [[OR19]], [[SHL20]]
- ; CHECK-NEXT: [[C22:%[0-9]+]]:_(s32) = G_CONSTANT i32 22
- ; CHECK-NEXT: [[SHL21:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C22]](s32)
- ; CHECK-NEXT: [[OR21:%[0-9]+]]:_(s32) = G_OR [[OR20]], [[SHL21]]
- ; CHECK-NEXT: [[C23:%[0-9]+]]:_(s32) = G_CONSTANT i32 23
- ; CHECK-NEXT: [[SHL22:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C23]](s32)
- ; CHECK-NEXT: [[OR22:%[0-9]+]]:_(s32) = G_OR [[OR21]], [[SHL22]]
- ; CHECK-NEXT: [[C24:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; CHECK-NEXT: [[SHL23:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C24]](s32)
- ; CHECK-NEXT: [[OR23:%[0-9]+]]:_(s32) = G_OR [[OR22]], [[SHL23]]
- ; CHECK-NEXT: [[C25:%[0-9]+]]:_(s32) = G_CONSTANT i32 25
- ; CHECK-NEXT: [[SHL24:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C25]](s32)
- ; CHECK-NEXT: [[OR24:%[0-9]+]]:_(s32) = G_OR [[OR23]], [[SHL24]]
- ; CHECK-NEXT: [[C26:%[0-9]+]]:_(s32) = G_CONSTANT i32 26
- ; CHECK-NEXT: [[SHL25:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C26]](s32)
- ; CHECK-NEXT: [[OR25:%[0-9]+]]:_(s32) = G_OR [[OR24]], [[SHL25]]
- ; CHECK-NEXT: [[C27:%[0-9]+]]:_(s32) = G_CONSTANT i32 27
- ; CHECK-NEXT: [[SHL26:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C27]](s32)
- ; CHECK-NEXT: [[OR26:%[0-9]+]]:_(s32) = G_OR [[OR25]], [[SHL26]]
- ; CHECK-NEXT: [[C28:%[0-9]+]]:_(s32) = G_CONSTANT i32 28
- ; CHECK-NEXT: [[SHL27:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C28]](s32)
- ; CHECK-NEXT: [[OR27:%[0-9]+]]:_(s32) = G_OR [[OR26]], [[SHL27]]
- ; CHECK-NEXT: [[C29:%[0-9]+]]:_(s32) = G_CONSTANT i32 29
- ; CHECK-NEXT: [[SHL28:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C29]](s32)
- ; CHECK-NEXT: [[OR28:%[0-9]+]]:_(s32) = G_OR [[OR27]], [[SHL28]]
- ; CHECK-NEXT: [[C30:%[0-9]+]]:_(s32) = G_CONSTANT i32 30
- ; CHECK-NEXT: [[SHL29:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C30]](s32)
- ; CHECK-NEXT: [[OR29:%[0-9]+]]:_(s32) = G_OR [[OR28]], [[SHL29]]
- ; CHECK-NEXT: [[C31:%[0-9]+]]:_(s32) = G_CONSTANT i32 31
- ; CHECK-NEXT: [[SHL30:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C31]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 2
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C3]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 3
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C4]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 4
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C5]](i32)
+ ; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[OR2]], [[SHL3]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 5
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C6]](i32)
+ ; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[OR3]], [[SHL4]]
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 6
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C7]](i32)
+ ; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[OR4]], [[SHL5]]
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 7
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C8]](i32)
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[OR5]], [[SHL6]]
+ ; CHECK-NEXT: [[C9:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C9]](i32)
+ ; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[OR6]], [[SHL7]]
+ ; CHECK-NEXT: [[C10:%[0-9]+]]:_(i32) = G_CONSTANT i32 9
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C10]](i32)
+ ; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[OR7]], [[SHL8]]
+ ; CHECK-NEXT: [[C11:%[0-9]+]]:_(i32) = G_CONSTANT i32 10
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C11]](i32)
+ ; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[OR8]], [[SHL9]]
+ ; CHECK-NEXT: [[C12:%[0-9]+]]:_(i32) = G_CONSTANT i32 11
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C12]](i32)
+ ; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[OR9]], [[SHL10]]
+ ; CHECK-NEXT: [[C13:%[0-9]+]]:_(i32) = G_CONSTANT i32 12
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C13]](i32)
+ ; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[OR10]], [[SHL11]]
+ ; CHECK-NEXT: [[C14:%[0-9]+]]:_(i32) = G_CONSTANT i32 13
+ ; CHECK-NEXT: [[SHL12:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C14]](i32)
+ ; CHECK-NEXT: [[OR12:%[0-9]+]]:_(i32) = G_OR [[OR11]], [[SHL12]]
+ ; CHECK-NEXT: [[C15:%[0-9]+]]:_(i32) = G_CONSTANT i32 14
+ ; CHECK-NEXT: [[SHL13:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C15]](i32)
+ ; CHECK-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[OR12]], [[SHL13]]
+ ; CHECK-NEXT: [[C16:%[0-9]+]]:_(i32) = G_CONSTANT i32 15
+ ; CHECK-NEXT: [[SHL14:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C16]](i32)
+ ; CHECK-NEXT: [[OR14:%[0-9]+]]:_(i32) = G_OR [[OR13]], [[SHL14]]
+ ; CHECK-NEXT: [[C17:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL15:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C17]](i32)
+ ; CHECK-NEXT: [[OR15:%[0-9]+]]:_(i32) = G_OR [[OR14]], [[SHL15]]
+ ; CHECK-NEXT: [[C18:%[0-9]+]]:_(i32) = G_CONSTANT i32 17
+ ; CHECK-NEXT: [[SHL16:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C18]](i32)
+ ; CHECK-NEXT: [[OR16:%[0-9]+]]:_(i32) = G_OR [[OR15]], [[SHL16]]
+ ; CHECK-NEXT: [[C19:%[0-9]+]]:_(i32) = G_CONSTANT i32 18
+ ; CHECK-NEXT: [[SHL17:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C19]](i32)
+ ; CHECK-NEXT: [[OR17:%[0-9]+]]:_(i32) = G_OR [[OR16]], [[SHL17]]
+ ; CHECK-NEXT: [[C20:%[0-9]+]]:_(i32) = G_CONSTANT i32 19
+ ; CHECK-NEXT: [[SHL18:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C20]](i32)
+ ; CHECK-NEXT: [[OR18:%[0-9]+]]:_(i32) = G_OR [[OR17]], [[SHL18]]
+ ; CHECK-NEXT: [[C21:%[0-9]+]]:_(i32) = G_CONSTANT i32 20
+ ; CHECK-NEXT: [[SHL19:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C21]](i32)
+ ; CHECK-NEXT: [[OR19:%[0-9]+]]:_(i32) = G_OR [[OR18]], [[SHL19]]
+ ; CHECK-NEXT: [[C22:%[0-9]+]]:_(i32) = G_CONSTANT i32 21
+ ; CHECK-NEXT: [[SHL20:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C22]](i32)
+ ; CHECK-NEXT: [[OR20:%[0-9]+]]:_(i32) = G_OR [[OR19]], [[SHL20]]
+ ; CHECK-NEXT: [[C23:%[0-9]+]]:_(i32) = G_CONSTANT i32 22
+ ; CHECK-NEXT: [[SHL21:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C23]](i32)
+ ; CHECK-NEXT: [[OR21:%[0-9]+]]:_(i32) = G_OR [[OR20]], [[SHL21]]
+ ; CHECK-NEXT: [[C24:%[0-9]+]]:_(i32) = G_CONSTANT i32 23
+ ; CHECK-NEXT: [[SHL22:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C24]](i32)
+ ; CHECK-NEXT: [[OR22:%[0-9]+]]:_(i32) = G_OR [[OR21]], [[SHL22]]
+ ; CHECK-NEXT: [[C25:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; CHECK-NEXT: [[SHL23:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C25]](i32)
+ ; CHECK-NEXT: [[OR23:%[0-9]+]]:_(i32) = G_OR [[OR22]], [[SHL23]]
+ ; CHECK-NEXT: [[C26:%[0-9]+]]:_(i32) = G_CONSTANT i32 25
+ ; CHECK-NEXT: [[SHL24:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C26]](i32)
+ ; CHECK-NEXT: [[OR24:%[0-9]+]]:_(i32) = G_OR [[OR23]], [[SHL24]]
+ ; CHECK-NEXT: [[C27:%[0-9]+]]:_(i32) = G_CONSTANT i32 26
+ ; CHECK-NEXT: [[SHL25:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C27]](i32)
+ ; CHECK-NEXT: [[OR25:%[0-9]+]]:_(i32) = G_OR [[OR24]], [[SHL25]]
+ ; CHECK-NEXT: [[C28:%[0-9]+]]:_(i32) = G_CONSTANT i32 27
+ ; CHECK-NEXT: [[SHL26:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C28]](i32)
+ ; CHECK-NEXT: [[OR26:%[0-9]+]]:_(i32) = G_OR [[OR25]], [[SHL26]]
+ ; CHECK-NEXT: [[C29:%[0-9]+]]:_(i32) = G_CONSTANT i32 28
+ ; CHECK-NEXT: [[SHL27:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C29]](i32)
+ ; CHECK-NEXT: [[OR27:%[0-9]+]]:_(i32) = G_OR [[OR26]], [[SHL27]]
+ ; CHECK-NEXT: [[C30:%[0-9]+]]:_(i32) = G_CONSTANT i32 29
+ ; CHECK-NEXT: [[SHL28:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C30]](i32)
+ ; CHECK-NEXT: [[OR28:%[0-9]+]]:_(i32) = G_OR [[OR27]], [[SHL28]]
+ ; CHECK-NEXT: [[C31:%[0-9]+]]:_(i32) = G_CONSTANT i32 30
+ ; CHECK-NEXT: [[SHL29:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C31]](i32)
+ ; CHECK-NEXT: [[OR29:%[0-9]+]]:_(i32) = G_OR [[OR28]], [[SHL29]]
+ ; CHECK-NEXT: [[C32:%[0-9]+]]:_(i32) = G_CONSTANT i32 31
+ ; CHECK-NEXT: [[SHL30:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C32]](i32)
; CHECK-NEXT: [[OR30:%[0-9]+]]:_(s32) = G_OR [[OR29]], [[SHL30]]
- ; CHECK-NEXT: [[SHL31:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; CHECK-NEXT: [[OR31:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL31]]
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY [[C]](s32)
- ; CHECK-NEXT: [[SHL32:%[0-9]+]]:_(s32) = G_SHL [[COPY]], [[C2]](s32)
- ; CHECK-NEXT: [[OR32:%[0-9]+]]:_(s32) = G_OR [[OR31]], [[SHL32]]
- ; CHECK-NEXT: [[SHL33:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
- ; CHECK-NEXT: [[OR33:%[0-9]+]]:_(s32) = G_OR [[OR32]], [[SHL33]]
- ; CHECK-NEXT: [[SHL34:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C4]](s32)
- ; CHECK-NEXT: [[OR34:%[0-9]+]]:_(s32) = G_OR [[OR33]], [[SHL34]]
- ; CHECK-NEXT: [[SHL35:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C5]](s32)
- ; CHECK-NEXT: [[OR35:%[0-9]+]]:_(s32) = G_OR [[OR34]], [[SHL35]]
- ; CHECK-NEXT: [[SHL36:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C6]](s32)
- ; CHECK-NEXT: [[OR36:%[0-9]+]]:_(s32) = G_OR [[OR35]], [[SHL36]]
- ; CHECK-NEXT: [[SHL37:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C7]](s32)
- ; CHECK-NEXT: [[OR37:%[0-9]+]]:_(s32) = G_OR [[OR36]], [[SHL37]]
- ; CHECK-NEXT: [[SHL38:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C8]](s32)
- ; CHECK-NEXT: [[OR38:%[0-9]+]]:_(s32) = G_OR [[OR37]], [[SHL38]]
- ; CHECK-NEXT: [[SHL39:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C9]](s32)
- ; CHECK-NEXT: [[OR39:%[0-9]+]]:_(s32) = G_OR [[OR38]], [[SHL39]]
- ; CHECK-NEXT: [[SHL40:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C10]](s32)
- ; CHECK-NEXT: [[OR40:%[0-9]+]]:_(s32) = G_OR [[OR39]], [[SHL40]]
- ; CHECK-NEXT: [[SHL41:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C11]](s32)
- ; CHECK-NEXT: [[OR41:%[0-9]+]]:_(s32) = G_OR [[OR40]], [[SHL41]]
- ; CHECK-NEXT: [[SHL42:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C12]](s32)
- ; CHECK-NEXT: [[OR42:%[0-9]+]]:_(s32) = G_OR [[OR41]], [[SHL42]]
- ; CHECK-NEXT: [[SHL43:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C13]](s32)
- ; CHECK-NEXT: [[OR43:%[0-9]+]]:_(s32) = G_OR [[OR42]], [[SHL43]]
- ; CHECK-NEXT: [[SHL44:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C14]](s32)
- ; CHECK-NEXT: [[OR44:%[0-9]+]]:_(s32) = G_OR [[OR43]], [[SHL44]]
- ; CHECK-NEXT: [[SHL45:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C15]](s32)
- ; CHECK-NEXT: [[OR45:%[0-9]+]]:_(s32) = G_OR [[OR44]], [[SHL45]]
- ; CHECK-NEXT: [[SHL46:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C16]](s32)
- ; CHECK-NEXT: [[OR46:%[0-9]+]]:_(s32) = G_OR [[OR45]], [[SHL46]]
- ; CHECK-NEXT: [[SHL47:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C17]](s32)
- ; CHECK-NEXT: [[OR47:%[0-9]+]]:_(s32) = G_OR [[OR46]], [[SHL47]]
- ; CHECK-NEXT: [[SHL48:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C18]](s32)
- ; CHECK-NEXT: [[OR48:%[0-9]+]]:_(s32) = G_OR [[OR47]], [[SHL48]]
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[C1]](s32)
- ; CHECK-NEXT: [[SHL49:%[0-9]+]]:_(s32) = G_SHL [[COPY1]], [[C19]](s32)
- ; CHECK-NEXT: [[OR49:%[0-9]+]]:_(s32) = G_OR [[OR48]], [[SHL49]]
- ; CHECK-NEXT: [[SHL50:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C20]](s32)
- ; CHECK-NEXT: [[OR50:%[0-9]+]]:_(s32) = G_OR [[OR49]], [[SHL50]]
- ; CHECK-NEXT: [[SHL51:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C21]](s32)
- ; CHECK-NEXT: [[OR51:%[0-9]+]]:_(s32) = G_OR [[OR50]], [[SHL51]]
- ; CHECK-NEXT: [[SHL52:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C22]](s32)
- ; CHECK-NEXT: [[OR52:%[0-9]+]]:_(s32) = G_OR [[OR51]], [[SHL52]]
- ; CHECK-NEXT: [[SHL53:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C23]](s32)
- ; CHECK-NEXT: [[OR53:%[0-9]+]]:_(s32) = G_OR [[OR52]], [[SHL53]]
- ; CHECK-NEXT: [[SHL54:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C24]](s32)
- ; CHECK-NEXT: [[OR54:%[0-9]+]]:_(s32) = G_OR [[OR53]], [[SHL54]]
- ; CHECK-NEXT: [[SHL55:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C25]](s32)
- ; CHECK-NEXT: [[OR55:%[0-9]+]]:_(s32) = G_OR [[OR54]], [[SHL55]]
- ; CHECK-NEXT: [[SHL56:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C26]](s32)
- ; CHECK-NEXT: [[OR56:%[0-9]+]]:_(s32) = G_OR [[OR55]], [[SHL56]]
- ; CHECK-NEXT: [[SHL57:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C27]](s32)
- ; CHECK-NEXT: [[OR57:%[0-9]+]]:_(s32) = G_OR [[OR56]], [[SHL57]]
- ; CHECK-NEXT: [[SHL58:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C28]](s32)
- ; CHECK-NEXT: [[OR58:%[0-9]+]]:_(s32) = G_OR [[OR57]], [[SHL58]]
- ; CHECK-NEXT: [[SHL59:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C29]](s32)
- ; CHECK-NEXT: [[OR59:%[0-9]+]]:_(s32) = G_OR [[OR58]], [[SHL59]]
- ; CHECK-NEXT: [[SHL60:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C30]](s32)
- ; CHECK-NEXT: [[OR60:%[0-9]+]]:_(s32) = G_OR [[OR59]], [[SHL60]]
- ; CHECK-NEXT: [[SHL61:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C31]](s32)
+ ; CHECK-NEXT: [[SHL31:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR31:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL31]]
+ ; CHECK-NEXT: [[C33:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(i32) = COPY [[C33]](i32)
+ ; CHECK-NEXT: [[SHL32:%[0-9]+]]:_(i32) = G_SHL [[COPY]], [[C3]](i32)
+ ; CHECK-NEXT: [[OR32:%[0-9]+]]:_(i32) = G_OR [[OR31]], [[SHL32]]
+ ; CHECK-NEXT: [[SHL33:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C4]](i32)
+ ; CHECK-NEXT: [[OR33:%[0-9]+]]:_(i32) = G_OR [[OR32]], [[SHL33]]
+ ; CHECK-NEXT: [[SHL34:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C5]](i32)
+ ; CHECK-NEXT: [[OR34:%[0-9]+]]:_(i32) = G_OR [[OR33]], [[SHL34]]
+ ; CHECK-NEXT: [[SHL35:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C6]](i32)
+ ; CHECK-NEXT: [[OR35:%[0-9]+]]:_(i32) = G_OR [[OR34]], [[SHL35]]
+ ; CHECK-NEXT: [[SHL36:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C7]](i32)
+ ; CHECK-NEXT: [[OR36:%[0-9]+]]:_(i32) = G_OR [[OR35]], [[SHL36]]
+ ; CHECK-NEXT: [[SHL37:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C8]](i32)
+ ; CHECK-NEXT: [[OR37:%[0-9]+]]:_(i32) = G_OR [[OR36]], [[SHL37]]
+ ; CHECK-NEXT: [[SHL38:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C9]](i32)
+ ; CHECK-NEXT: [[OR38:%[0-9]+]]:_(i32) = G_OR [[OR37]], [[SHL38]]
+ ; CHECK-NEXT: [[SHL39:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C10]](i32)
+ ; CHECK-NEXT: [[OR39:%[0-9]+]]:_(i32) = G_OR [[OR38]], [[SHL39]]
+ ; CHECK-NEXT: [[SHL40:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C11]](i32)
+ ; CHECK-NEXT: [[OR40:%[0-9]+]]:_(i32) = G_OR [[OR39]], [[SHL40]]
+ ; CHECK-NEXT: [[SHL41:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C12]](i32)
+ ; CHECK-NEXT: [[OR41:%[0-9]+]]:_(i32) = G_OR [[OR40]], [[SHL41]]
+ ; CHECK-NEXT: [[SHL42:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C13]](i32)
+ ; CHECK-NEXT: [[OR42:%[0-9]+]]:_(i32) = G_OR [[OR41]], [[SHL42]]
+ ; CHECK-NEXT: [[SHL43:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C14]](i32)
+ ; CHECK-NEXT: [[OR43:%[0-9]+]]:_(i32) = G_OR [[OR42]], [[SHL43]]
+ ; CHECK-NEXT: [[SHL44:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C15]](i32)
+ ; CHECK-NEXT: [[OR44:%[0-9]+]]:_(i32) = G_OR [[OR43]], [[SHL44]]
+ ; CHECK-NEXT: [[SHL45:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C16]](i32)
+ ; CHECK-NEXT: [[OR45:%[0-9]+]]:_(i32) = G_OR [[OR44]], [[SHL45]]
+ ; CHECK-NEXT: [[SHL46:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C17]](i32)
+ ; CHECK-NEXT: [[OR46:%[0-9]+]]:_(i32) = G_OR [[OR45]], [[SHL46]]
+ ; CHECK-NEXT: [[SHL47:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C18]](i32)
+ ; CHECK-NEXT: [[OR47:%[0-9]+]]:_(i32) = G_OR [[OR46]], [[SHL47]]
+ ; CHECK-NEXT: [[SHL48:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C19]](i32)
+ ; CHECK-NEXT: [[OR48:%[0-9]+]]:_(i32) = G_OR [[OR47]], [[SHL48]]
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i32) = COPY [[C2]](i32)
+ ; CHECK-NEXT: [[SHL49:%[0-9]+]]:_(i32) = G_SHL [[COPY1]], [[C20]](i32)
+ ; CHECK-NEXT: [[OR49:%[0-9]+]]:_(i32) = G_OR [[OR48]], [[SHL49]]
+ ; CHECK-NEXT: [[SHL50:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C21]](i32)
+ ; CHECK-NEXT: [[OR50:%[0-9]+]]:_(i32) = G_OR [[OR49]], [[SHL50]]
+ ; CHECK-NEXT: [[SHL51:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C22]](i32)
+ ; CHECK-NEXT: [[OR51:%[0-9]+]]:_(i32) = G_OR [[OR50]], [[SHL51]]
+ ; CHECK-NEXT: [[SHL52:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C23]](i32)
+ ; CHECK-NEXT: [[OR52:%[0-9]+]]:_(i32) = G_OR [[OR51]], [[SHL52]]
+ ; CHECK-NEXT: [[SHL53:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C24]](i32)
+ ; CHECK-NEXT: [[OR53:%[0-9]+]]:_(i32) = G_OR [[OR52]], [[SHL53]]
+ ; CHECK-NEXT: [[SHL54:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C25]](i32)
+ ; CHECK-NEXT: [[OR54:%[0-9]+]]:_(i32) = G_OR [[OR53]], [[SHL54]]
+ ; CHECK-NEXT: [[SHL55:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C26]](i32)
+ ; CHECK-NEXT: [[OR55:%[0-9]+]]:_(i32) = G_OR [[OR54]], [[SHL55]]
+ ; CHECK-NEXT: [[SHL56:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C27]](i32)
+ ; CHECK-NEXT: [[OR56:%[0-9]+]]:_(i32) = G_OR [[OR55]], [[SHL56]]
+ ; CHECK-NEXT: [[SHL57:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C28]](i32)
+ ; CHECK-NEXT: [[OR57:%[0-9]+]]:_(i32) = G_OR [[OR56]], [[SHL57]]
+ ; CHECK-NEXT: [[SHL58:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C29]](i32)
+ ; CHECK-NEXT: [[OR58:%[0-9]+]]:_(i32) = G_OR [[OR57]], [[SHL58]]
+ ; CHECK-NEXT: [[SHL59:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C30]](i32)
+ ; CHECK-NEXT: [[OR59:%[0-9]+]]:_(i32) = G_OR [[OR58]], [[SHL59]]
+ ; CHECK-NEXT: [[SHL60:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C31]](i32)
+ ; CHECK-NEXT: [[OR60:%[0-9]+]]:_(i32) = G_OR [[OR59]], [[SHL60]]
+ ; CHECK-NEXT: [[SHL61:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C32]](i32)
; CHECK-NEXT: [[OR61:%[0-9]+]]:_(s32) = G_OR [[OR60]], [[SHL61]]
- ; CHECK-NEXT: [[SHL62:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; CHECK-NEXT: [[OR62:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL62]]
- ; CHECK-NEXT: [[SHL63:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C2]](s32)
- ; CHECK-NEXT: [[OR63:%[0-9]+]]:_(s32) = G_OR [[OR62]], [[SHL63]]
- ; CHECK-NEXT: [[SHL64:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C3]](s32)
- ; CHECK-NEXT: [[OR64:%[0-9]+]]:_(s32) = G_OR [[OR63]], [[SHL64]]
- ; CHECK-NEXT: [[SHL65:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C4]](s32)
- ; CHECK-NEXT: [[OR65:%[0-9]+]]:_(s32) = G_OR [[OR64]], [[SHL65]]
- ; CHECK-NEXT: [[SHL66:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C5]](s32)
- ; CHECK-NEXT: [[OR66:%[0-9]+]]:_(s32) = G_OR [[OR65]], [[SHL66]]
- ; CHECK-NEXT: [[SHL67:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C6]](s32)
- ; CHECK-NEXT: [[OR67:%[0-9]+]]:_(s32) = G_OR [[OR66]], [[SHL67]]
- ; CHECK-NEXT: [[SHL68:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C7]](s32)
- ; CHECK-NEXT: [[OR68:%[0-9]+]]:_(s32) = G_OR [[OR67]], [[SHL68]]
- ; CHECK-NEXT: [[SHL69:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C8]](s32)
- ; CHECK-NEXT: [[OR69:%[0-9]+]]:_(s32) = G_OR [[OR68]], [[SHL69]]
- ; CHECK-NEXT: [[SHL70:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C9]](s32)
- ; CHECK-NEXT: [[OR70:%[0-9]+]]:_(s32) = G_OR [[OR69]], [[SHL70]]
- ; CHECK-NEXT: [[SHL71:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C10]](s32)
- ; CHECK-NEXT: [[OR71:%[0-9]+]]:_(s32) = G_OR [[OR70]], [[SHL71]]
- ; CHECK-NEXT: [[SHL72:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C11]](s32)
- ; CHECK-NEXT: [[OR72:%[0-9]+]]:_(s32) = G_OR [[OR71]], [[SHL72]]
- ; CHECK-NEXT: [[SHL73:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C12]](s32)
- ; CHECK-NEXT: [[OR73:%[0-9]+]]:_(s32) = G_OR [[OR72]], [[SHL73]]
- ; CHECK-NEXT: [[SHL74:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C13]](s32)
- ; CHECK-NEXT: [[OR74:%[0-9]+]]:_(s32) = G_OR [[OR73]], [[SHL74]]
- ; CHECK-NEXT: [[SHL75:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C14]](s32)
- ; CHECK-NEXT: [[OR75:%[0-9]+]]:_(s32) = G_OR [[OR74]], [[SHL75]]
- ; CHECK-NEXT: [[SHL76:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C15]](s32)
- ; CHECK-NEXT: [[OR76:%[0-9]+]]:_(s32) = G_OR [[OR75]], [[SHL76]]
- ; CHECK-NEXT: [[SHL77:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C16]](s32)
- ; CHECK-NEXT: [[OR77:%[0-9]+]]:_(s32) = G_OR [[OR76]], [[SHL77]]
- ; CHECK-NEXT: [[SHL78:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C17]](s32)
- ; CHECK-NEXT: [[OR78:%[0-9]+]]:_(s32) = G_OR [[OR77]], [[SHL78]]
- ; CHECK-NEXT: [[SHL79:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C18]](s32)
- ; CHECK-NEXT: [[OR79:%[0-9]+]]:_(s32) = G_OR [[OR78]], [[SHL79]]
- ; CHECK-NEXT: [[SHL80:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C19]](s32)
- ; CHECK-NEXT: [[OR80:%[0-9]+]]:_(s32) = G_OR [[OR79]], [[SHL80]]
- ; CHECK-NEXT: [[SHL81:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C20]](s32)
- ; CHECK-NEXT: [[OR81:%[0-9]+]]:_(s32) = G_OR [[OR80]], [[SHL81]]
- ; CHECK-NEXT: [[SHL82:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C21]](s32)
- ; CHECK-NEXT: [[OR82:%[0-9]+]]:_(s32) = G_OR [[OR81]], [[SHL82]]
- ; CHECK-NEXT: [[SHL83:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C22]](s32)
- ; CHECK-NEXT: [[OR83:%[0-9]+]]:_(s32) = G_OR [[OR82]], [[SHL83]]
- ; CHECK-NEXT: [[SHL84:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C23]](s32)
- ; CHECK-NEXT: [[OR84:%[0-9]+]]:_(s32) = G_OR [[OR83]], [[SHL84]]
- ; CHECK-NEXT: [[SHL85:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C24]](s32)
- ; CHECK-NEXT: [[OR85:%[0-9]+]]:_(s32) = G_OR [[OR84]], [[SHL85]]
- ; CHECK-NEXT: [[SHL86:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C25]](s32)
- ; CHECK-NEXT: [[OR86:%[0-9]+]]:_(s32) = G_OR [[OR85]], [[SHL86]]
- ; CHECK-NEXT: [[SHL87:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C26]](s32)
- ; CHECK-NEXT: [[OR87:%[0-9]+]]:_(s32) = G_OR [[OR86]], [[SHL87]]
- ; CHECK-NEXT: [[SHL88:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C27]](s32)
- ; CHECK-NEXT: [[OR88:%[0-9]+]]:_(s32) = G_OR [[OR87]], [[SHL88]]
- ; CHECK-NEXT: [[SHL89:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C28]](s32)
- ; CHECK-NEXT: [[OR89:%[0-9]+]]:_(s32) = G_OR [[OR88]], [[SHL89]]
- ; CHECK-NEXT: [[SHL90:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C29]](s32)
- ; CHECK-NEXT: [[OR90:%[0-9]+]]:_(s32) = G_OR [[OR89]], [[SHL90]]
- ; CHECK-NEXT: [[SHL91:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C30]](s32)
- ; CHECK-NEXT: [[OR91:%[0-9]+]]:_(s32) = G_OR [[OR90]], [[SHL91]]
- ; CHECK-NEXT: [[SHL92:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C31]](s32)
+ ; CHECK-NEXT: [[SHL62:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR62:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL62]]
+ ; CHECK-NEXT: [[SHL63:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C3]](i32)
+ ; CHECK-NEXT: [[OR63:%[0-9]+]]:_(i32) = G_OR [[OR62]], [[SHL63]]
+ ; CHECK-NEXT: [[SHL64:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C4]](i32)
+ ; CHECK-NEXT: [[OR64:%[0-9]+]]:_(i32) = G_OR [[OR63]], [[SHL64]]
+ ; CHECK-NEXT: [[SHL65:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C5]](i32)
+ ; CHECK-NEXT: [[OR65:%[0-9]+]]:_(i32) = G_OR [[OR64]], [[SHL65]]
+ ; CHECK-NEXT: [[SHL66:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C6]](i32)
+ ; CHECK-NEXT: [[OR66:%[0-9]+]]:_(i32) = G_OR [[OR65]], [[SHL66]]
+ ; CHECK-NEXT: [[SHL67:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C7]](i32)
+ ; CHECK-NEXT: [[OR67:%[0-9]+]]:_(i32) = G_OR [[OR66]], [[SHL67]]
+ ; CHECK-NEXT: [[SHL68:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C8]](i32)
+ ; CHECK-NEXT: [[OR68:%[0-9]+]]:_(i32) = G_OR [[OR67]], [[SHL68]]
+ ; CHECK-NEXT: [[SHL69:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C9]](i32)
+ ; CHECK-NEXT: [[OR69:%[0-9]+]]:_(i32) = G_OR [[OR68]], [[SHL69]]
+ ; CHECK-NEXT: [[SHL70:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C10]](i32)
+ ; CHECK-NEXT: [[OR70:%[0-9]+]]:_(i32) = G_OR [[OR69]], [[SHL70]]
+ ; CHECK-NEXT: [[SHL71:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C11]](i32)
+ ; CHECK-NEXT: [[OR71:%[0-9]+]]:_(i32) = G_OR [[OR70]], [[SHL71]]
+ ; CHECK-NEXT: [[SHL72:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C12]](i32)
+ ; CHECK-NEXT: [[OR72:%[0-9]+]]:_(i32) = G_OR [[OR71]], [[SHL72]]
+ ; CHECK-NEXT: [[SHL73:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C13]](i32)
+ ; CHECK-NEXT: [[OR73:%[0-9]+]]:_(i32) = G_OR [[OR72]], [[SHL73]]
+ ; CHECK-NEXT: [[SHL74:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C14]](i32)
+ ; CHECK-NEXT: [[OR74:%[0-9]+]]:_(i32) = G_OR [[OR73]], [[SHL74]]
+ ; CHECK-NEXT: [[SHL75:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C15]](i32)
+ ; CHECK-NEXT: [[OR75:%[0-9]+]]:_(i32) = G_OR [[OR74]], [[SHL75]]
+ ; CHECK-NEXT: [[SHL76:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C16]](i32)
+ ; CHECK-NEXT: [[OR76:%[0-9]+]]:_(i32) = G_OR [[OR75]], [[SHL76]]
+ ; CHECK-NEXT: [[SHL77:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C17]](i32)
+ ; CHECK-NEXT: [[OR77:%[0-9]+]]:_(i32) = G_OR [[OR76]], [[SHL77]]
+ ; CHECK-NEXT: [[SHL78:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C18]](i32)
+ ; CHECK-NEXT: [[OR78:%[0-9]+]]:_(i32) = G_OR [[OR77]], [[SHL78]]
+ ; CHECK-NEXT: [[SHL79:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C19]](i32)
+ ; CHECK-NEXT: [[OR79:%[0-9]+]]:_(i32) = G_OR [[OR78]], [[SHL79]]
+ ; CHECK-NEXT: [[SHL80:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C20]](i32)
+ ; CHECK-NEXT: [[OR80:%[0-9]+]]:_(i32) = G_OR [[OR79]], [[SHL80]]
+ ; CHECK-NEXT: [[SHL81:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C21]](i32)
+ ; CHECK-NEXT: [[OR81:%[0-9]+]]:_(i32) = G_OR [[OR80]], [[SHL81]]
+ ; CHECK-NEXT: [[SHL82:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C22]](i32)
+ ; CHECK-NEXT: [[OR82:%[0-9]+]]:_(i32) = G_OR [[OR81]], [[SHL82]]
+ ; CHECK-NEXT: [[SHL83:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C23]](i32)
+ ; CHECK-NEXT: [[OR83:%[0-9]+]]:_(i32) = G_OR [[OR82]], [[SHL83]]
+ ; CHECK-NEXT: [[SHL84:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C24]](i32)
+ ; CHECK-NEXT: [[OR84:%[0-9]+]]:_(i32) = G_OR [[OR83]], [[SHL84]]
+ ; CHECK-NEXT: [[SHL85:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C25]](i32)
+ ; CHECK-NEXT: [[OR85:%[0-9]+]]:_(i32) = G_OR [[OR84]], [[SHL85]]
+ ; CHECK-NEXT: [[SHL86:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C26]](i32)
+ ; CHECK-NEXT: [[OR86:%[0-9]+]]:_(i32) = G_OR [[OR85]], [[SHL86]]
+ ; CHECK-NEXT: [[SHL87:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C27]](i32)
+ ; CHECK-NEXT: [[OR87:%[0-9]+]]:_(i32) = G_OR [[OR86]], [[SHL87]]
+ ; CHECK-NEXT: [[SHL88:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C28]](i32)
+ ; CHECK-NEXT: [[OR88:%[0-9]+]]:_(i32) = G_OR [[OR87]], [[SHL88]]
+ ; CHECK-NEXT: [[SHL89:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C29]](i32)
+ ; CHECK-NEXT: [[OR89:%[0-9]+]]:_(i32) = G_OR [[OR88]], [[SHL89]]
+ ; CHECK-NEXT: [[SHL90:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C30]](i32)
+ ; CHECK-NEXT: [[OR90:%[0-9]+]]:_(i32) = G_OR [[OR89]], [[SHL90]]
+ ; CHECK-NEXT: [[SHL91:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C31]](i32)
+ ; CHECK-NEXT: [[OR91:%[0-9]+]]:_(i32) = G_OR [[OR90]], [[SHL91]]
+ ; CHECK-NEXT: [[SHL92:%[0-9]+]]:_(i32) = G_SHL [[C33]], [[C32]](i32)
; CHECK-NEXT: [[OR92:%[0-9]+]]:_(s32) = G_OR [[OR91]], [[SHL92]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s96) = G_MERGE_VALUES [[OR30]](s32), [[OR61]](s32), [[OR92]](s32)
; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s68) = G_TRUNC [[MV]](s96)
@@ -834,12 +837,12 @@ name: test_merge_p3_s16_s16
body: |
bb.0:
; CHECK-LABEL: name: test_merge_p3_s16_s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[C]], [[SHL]]
- ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; CHECK-NEXT: [[INTTOPTR:%[0-9]+]]:_(p3) = G_INTTOPTR [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[INTTOPTR]](p3)
%0:_(s16) = G_CONSTANT i16 0
%1:_(s16) = G_CONSTANT i16 1
@@ -858,11 +861,11 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: $vgpr0 = COPY [[OR]](s32)
%0:_(s32) = COPY $vgpr0
@@ -885,15 +888,15 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[COPY1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](s32), [[OR1]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[MV]](s64)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
index ffc7f6dfeb94e..e514c11cbc762 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-mul.mir
@@ -279,11 +279,10 @@ body: |
; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[BITCAST]], [[BITCAST1]]
; GFX6-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[LSHR1]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -304,10 +303,9 @@ body: |
; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[MUL:%[0-9]+]]:_(s16) = G_MUL [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s16) = G_MUL [[TRUNC1]], [[TRUNC3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[MUL]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[MUL1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[MUL]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[MUL1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -442,16 +440,15 @@ body: |
; GFX6-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[LSHR]], [[LSHR2]]
; GFX6-NEXT: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[BITCAST1]], [[BITCAST3]]
; GFX6-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[LSHR1]], [[LSHR3]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[MUL2]], [[C1]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[MUL3]], [[C1]]
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -485,15 +482,14 @@ body: |
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s16) = G_MUL [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[MUL2:%[0-9]+]]:_(s16) = G_MUL [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[MUL3:%[0-9]+]]:_(s16) = G_MUL [[TRUNC3]], [[TRUNC7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[MUL]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[MUL1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[MUL]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[MUL1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[MUL2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[MUL3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[MUL2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[MUL3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
index 2fa7f167aabc9..46f71c81a9301 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-or.mir
@@ -456,9 +456,9 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV3]](<2 x s16>), [[UV4]](<2 x s16>)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(<4 x s16>) = G_OR [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[OR]](<4 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV6]](<2 x s16>), [[UV7]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -499,60 +499,59 @@ body: |
bb.0:
; CHECK-LABEL: name: test_or_v5s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(<4 x s16>) = G_OR [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[OR4]](<4 x s16>)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C2]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL4]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL5]]
; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL6]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL7]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(<4 x s16>) = G_OR [[CONCAT_VECTORS2]], [[CONCAT_VECTORS3]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[OR9]](<4 x s16>)
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C2]](i32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C3]]
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST4]], [[C2]]
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C3]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C3]]
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST5]], [[C2]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST10]], [[C2]]
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C1]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL9]]
; CHECK-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](i32)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR12:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL10]]
; CHECK-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR12]](i32)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR13:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL11]]
; CHECK-NEXT: [[BITCAST14:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR13]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS4:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST11]](<2 x s16>), [[BITCAST12]](<2 x s16>), [[BITCAST13]](<2 x s16>), [[BITCAST14]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
index 72db13a0fc190..b7a88cdf80833 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-phi.mir
@@ -78,11 +78,10 @@ body: |
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C2]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: G_BR %bb.2
@@ -133,10 +132,10 @@ body: |
; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C2]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -149,16 +148,16 @@ body: |
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST1]]
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C4]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C4]]
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C5]](s32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C4]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C4]]
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C5]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C4]]
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C5]](s32)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C4]]
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C5]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL2]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -167,9 +166,9 @@ body: |
; CHECK-NEXT: bb.2:
; CHECK-NEXT: [[PHI:%[0-9]+]]:_(<4 x s16>) = G_PHI [[CONCAT_VECTORS]](<4 x s16>), %bb.0, [[CONCAT_VECTORS1]](<4 x s16>), %bb.1
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[PHI]](<4 x s16>)
- ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C7]], [[C8]](s32)
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C7]], [[C8]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL3]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV2]](<2 x s16>), [[UV3]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -236,16 +235,15 @@ body: |
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR2]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR1]], [[LSHR3]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C2]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C2]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C2]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ADD3]], [[C2]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir
index 4747abfc6c867..9fe94a05286bb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddo.mir
@@ -147,11 +147,10 @@ body: |
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[BITCAST]], [[BITCAST1]]
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
@@ -165,19 +164,19 @@ body: |
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[COPY1]](<2 x s16>)
; CHECK-NEXT: [[LSHR3:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST4]], 16
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG4]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR3]], 16
- ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C3]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C2]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP2]], [[ICMP]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP3]], [[ICMP1]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY [[BITCAST2]](<2 x s16>)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND2]](s32), [[AND3]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[COPY3]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -243,19 +242,19 @@ body: |
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1)
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C2]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>), [[BITCAST10]](<2 x s16>)
; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
@@ -303,16 +302,15 @@ body: |
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR2]]
; CHECK-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[BITCAST1]], [[BITCAST3]]
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR1]], [[LSHR3]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ADD2]], [[C1]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C1]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ADD3]], [[C1]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -339,17 +337,17 @@ body: |
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(i32) = G_BITCAST [[UV7]](<2 x s16>)
; CHECK-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST9]], [[C]](i32)
; CHECK-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST8]], 16
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG8]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR6]], 16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG9]](s32), [[COPY3]]
; CHECK-NEXT: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST9]], 16
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG10]](s32), [[COPY4]]
; CHECK-NEXT: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR7]], 16
- ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C3]]
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C2]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP4]], [[ICMP]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP5]], [[ICMP1]]
; CHECK-NEXT: [[XOR2:%[0-9]+]]:_(s1) = G_XOR [[ICMP6]], [[ICMP2]]
@@ -359,11 +357,11 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR3]](s1)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(<4 x s16>) = COPY [[CONCAT_VECTORS]](<4 x s16>)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C4]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32), [[AND6]](s32), [[AND7]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY5]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
index 152399a57dc5a..d977202256095 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-saddsat.mir
@@ -397,10 +397,10 @@ body: |
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB2]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[SMIN3]]
; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[ADD1]], [[C1]](s32)
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -436,10 +436,9 @@ body: |
; GFX8-NEXT: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC3]]
; GFX8-NEXT: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB2]]
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[SMIN3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -509,17 +508,18 @@ body: |
; GFX6-NEXT: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB4]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[SMIN5]]
; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[ADD2]], [[C1]](s32)
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -566,18 +566,17 @@ body: |
; GFX8-NEXT: [[SMAX5:%[0-9]+]]:_(s16) = G_SMAX [[SUB5]], [[TRUNC5]]
; GFX8-NEXT: [[SMIN5:%[0-9]+]]:_(s16) = G_SMIN [[SMAX5]], [[SUB4]]
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[SMIN5]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -685,15 +684,15 @@ body: |
; GFX6-NEXT: [[SMIN7:%[0-9]+]]:_(s32) = G_SMIN [[SMAX7]], [[SUB6]]
; GFX6-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SHL6]], [[SMIN7]]
; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[ADD3]], [[C1]](s32)
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C5]]
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ASHR3]], [[C5]]
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -754,15 +753,14 @@ body: |
; GFX8-NEXT: [[SMAX7:%[0-9]+]]:_(s16) = G_SMAX [[SUB7]], [[TRUNC7]]
; GFX8-NEXT: [[SMIN7:%[0-9]+]]:_(s16) = G_SMIN [[SMAX7]], [[SUB6]]
; GFX8-NEXT: [[ADD3:%[0-9]+]]:_(s16) = G_ADD [[TRUNC3]], [[SMIN7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[ADD3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[ADD3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
index e6b820dcab5e2..9777b06041fa3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sdiv.mir
@@ -2771,11 +2771,10 @@ body: |
; GFX6-NEXT: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
; GFX6-NEXT: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB7]], [[C5]]
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB3]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB7]], [[C5]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2854,11 +2853,10 @@ body: |
; GFX8-NEXT: [[XOR6:%[0-9]+]]:_(s32) = G_XOR [[ASHR2]], [[ASHR3]]
; GFX8-NEXT: [[XOR7:%[0-9]+]]:_(s32) = G_XOR [[SELECT5]], [[XOR6]]
; GFX8-NEXT: [[SUB7:%[0-9]+]]:_(s32) = G_SUB [[XOR7]], [[XOR6]]
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C5]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB7]], [[C5]]
- ; GFX8-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C6]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB3]], [[C5]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB7]], [[C5]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
index d428d7868173b..a263bec4abd64 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-select.mir
@@ -406,9 +406,10 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV3]](<2 x s16>), [[UV4]](<2 x s16>)
; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(<4 x s16>) = G_SELECT [[ICMP]](s1), [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[SELECT]](<4 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV6]](<2 x s16>), [[UV7]](<2 x s16>), [[BITCAST]](<2 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS2]](<6 x s16>)
@@ -1331,19 +1332,18 @@ body: |
; CHECK-NEXT: [[SELECT:%[0-9]+]]:_(s16) = G_SELECT [[ICMP]](s1), [[TRUNC]], [[TRUNC3]]
; CHECK-NEXT: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[ICMP1]](s1), [[TRUNC1]], [[TRUNC4]]
; CHECK-NEXT: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP2]](s1), [[TRUNC2]], [[TRUNC5]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT]](s16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT2]](s16)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT2]](s16)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
index 2e3c0bf0b1258..ca42ff061391f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext-inreg.mir
@@ -885,11 +885,10 @@ body: |
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C1]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C1]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -903,11 +902,10 @@ body: |
; GFX6-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C]](i32)
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -960,20 +958,19 @@ body: |
; GFX8-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX8-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C1]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C1]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
- ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -990,20 +987,19 @@ body: |
; GFX6-NEXT: [[SEXT_INREG:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST]], 1
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX6-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C1]]
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL2]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -1139,16 +1135,15 @@ body: |
; GFX8-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX8-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
; GFX8-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 1
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C1]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C1]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
- ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C1]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C1]]
+ ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C1]]
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1168,16 +1163,15 @@ body: |
; GFX6-NEXT: [[SEXT_INREG1:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 1
; GFX6-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 1
; GFX6-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 1
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C1]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C1]]
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -1215,21 +1209,21 @@ body: |
; GFX8-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[DEF]], 1
; GFX8-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[DEF]], 1
; GFX8-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[DEF]], 1
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C]]
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
- ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C]]
+ ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C]]
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG4]], [[C]]
- ; GFX8-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG5]], [[C]]
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX8-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG4]], [[C]]
+ ; GFX8-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG5]], [[C]]
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
@@ -1243,21 +1237,21 @@ body: |
; GFX6-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[DEF]], 1
; GFX6-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[DEF]], 1
; GFX6-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[DEF]], 1
- ; GFX6-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG]], [[C]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG1]], [[C]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG]], [[C]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG1]], [[C]]
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG2]], [[C]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG3]], [[C]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG2]], [[C]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG3]], [[C]]
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG4]], [[C]]
- ; GFX6-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SEXT_INREG5]], [[C]]
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG4]], [[C]]
+ ; GFX6-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SEXT_INREG5]], [[C]]
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C1]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL2]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
index 6f69f90e0ea82..13e7ef34292b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sext.mir
@@ -677,13 +677,14 @@ body: |
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i16) = G_AND [[ASHR]], [[C4]]
; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i16) = G_SHL [[AND7]], [[C5]](i16)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i16) = G_OR [[AND6]], [[SHL3]]
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](i16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](i16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](i16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](i16)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C6]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](i16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[OR3]](i16)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](i16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[OR3]](i16)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C6]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR4]](i32), [[OR5]](i32)
; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i16) = G_AND [[ASHR]], [[C4]]
@@ -702,13 +703,13 @@ body: |
; CHECK-NEXT: [[AND15:%[0-9]+]]:_(i16) = G_AND [[ASHR]], [[C4]]
; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i16) = G_SHL [[AND15]], [[C5]](i16)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i16) = G_OR [[AND14]], [[SHL9]]
- ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[OR6]](i16)
- ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(s32) = G_ZEXT [[OR7]](i16)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[ZEXT5]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[OR6]](i16)
+ ; CHECK-NEXT: [[ZEXT5:%[0-9]+]]:_(i32) = G_ZEXT [[OR7]](i16)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[ZEXT5]], [[C6]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL10]]
- ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(s32) = G_ZEXT [[OR8]](i16)
- ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(s32) = G_ZEXT [[OR9]](i16)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[ZEXT7]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT6:%[0-9]+]]:_(i32) = G_ZEXT [[OR8]](i16)
+ ; CHECK-NEXT: [[ZEXT7:%[0-9]+]]:_(i32) = G_ZEXT [[OR9]](i16)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[ZEXT7]], [[C6]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[ZEXT6]], [[SHL11]]
; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR10]](i32), [[OR11]](i32)
; CHECK-NEXT: [[MV2:%[0-9]+]]:_(s704) = G_MERGE_VALUES [[MV]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64), [[MV1]](s64)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
index e72b9aed49f5b..f01791b61d935 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shl.mir
@@ -562,10 +562,10 @@ body: |
; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[AND]](s32)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR1]](i32)
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SHL]], [[C2]]
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SHL1]], [[C2]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -586,10 +586,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SHL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SHL1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -624,11 +623,10 @@ body: |
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<2 x s32>)
; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[BITCAST]], [[UV]](s32)
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[UV1]](s32)
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SHL]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SHL1]], [[C1]]
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL2]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -648,10 +646,9 @@ body: |
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC2]](s16)
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC3]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SHL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SHL1]](s16)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -704,18 +701,18 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[LSHR1]](i32)
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[AND1]](s32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SHL]], [[C2]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SHL1]], [[C2]]
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL3]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C1]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SHL2]], [[C2]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL4]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL5]]
; SI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
@@ -744,19 +741,18 @@ body: |
; VI-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC4]](s16)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC5]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SHL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SHL1]](s16)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SHL2]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SHL2]](s16)
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
- ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL5]]
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
+ ; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL5]]
; VI-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; VI-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -827,15 +823,15 @@ body: |
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[BITCAST1]], [[AND1]](s32)
; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[LSHR3]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SHL]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SHL1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SHL]], [[C2]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SHL1]], [[C2]]
+ ; SI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL4]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[SHL2]], [[C1]]
- ; SI-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[SHL3]], [[C1]]
- ; SI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SHL2]], [[C2]]
+ ; SI-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SHL3]], [[C2]]
+ ; SI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL5]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -869,15 +865,14 @@ body: |
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC5]](s16)
; VI-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC6]](s16)
; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[TRUNC3]], [[TRUNC7]](s16)
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SHL]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SHL1]](s16)
+ ; VI-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SHL2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SHL3]](s16)
- ; VI-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SHL2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SHL3]](s16)
+ ; VI-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
index dc6e35313d22e..61838b2195379 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.mir
@@ -241,9 +241,10 @@ body: |
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C3]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -273,9 +274,10 @@ body: |
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -306,15 +308,14 @@ body: |
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C1]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
index 73e1b61d32597..3f858543f4692 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-shuffle-vector.s16.mir
@@ -17,9 +17,9 @@ body: |
; GFX8-NEXT: {{ $}}
; GFX8-NEXT: [[COPY:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr0
; GFX8-NEXT: [[COPY1:%[0-9]+]]:_(<2 x s16>) = COPY $vgpr1
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -55,10 +55,11 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -98,11 +99,12 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
- ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
+ ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
;
@@ -144,9 +146,10 @@ body: |
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -192,9 +195,10 @@ body: |
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -237,10 +241,10 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -279,7 +283,8 @@ body: |
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -318,10 +323,10 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -360,10 +365,11 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -403,11 +409,12 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
- ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
+ ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
;
@@ -449,9 +456,10 @@ body: |
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -497,9 +505,10 @@ body: |
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -542,8 +551,9 @@ body: |
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -583,8 +593,9 @@ body: |
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -624,7 +635,8 @@ body: |
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -665,11 +677,11 @@ body: |
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -711,10 +723,10 @@ body: |
; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>)
@@ -756,9 +768,10 @@ body: |
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -804,9 +817,10 @@ body: |
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -852,9 +866,10 @@ body: |
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C1]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY1]](<2 x s16>)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -900,9 +915,10 @@ body: |
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>)
; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST1]], [[C1]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -949,15 +965,14 @@ body: |
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[COPY1]](<4 x s16>)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(i32) = G_BITCAST [[UV3]](<2 x s16>)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST2]], [[C1]]
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C2]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST2]], [[C1]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C1]]
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C1]]
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>)
@@ -1015,9 +1030,10 @@ body: |
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s32>)
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV2]](s32)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[LSHR1]], [[C2]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[LSHR1]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND]], [[C3]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -1070,7 +1086,8 @@ body: |
; GFX8-NEXT: [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST1]](<2 x s32>)
; GFX8-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[UV3]](s32)
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[COPY3]], [[C]](s32)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
index f4aeaf43066bf..e7fb600e62b1c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sitofp.mir
@@ -631,10 +631,10 @@ body: |
; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[C]], [[UMIN2]]
; GFX6-NEXT: [[FLDEXP1:%[0-9]+]]:_(s32) = G_FLDEXP [[SITOFP1]], [[SUB3]](i32)
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FLDEXP1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -677,10 +677,10 @@ body: |
; GFX8-NEXT: [[SUB3:%[0-9]+]]:_(i32) = G_SUB [[C]], [[UMIN2]]
; GFX8-NEXT: [[FLDEXP1:%[0-9]+]]:_(s32) = G_FLDEXP [[SITOFP1]], [[SUB3]](i32)
; GFX8-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FLDEXP1]](s32)
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
index a0518478814a8..3603bb6464014 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smax.mir
@@ -413,11 +413,10 @@ body: |
; SI-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; SI-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[SMAX1:%[0-9]+]]:_(s32) = G_SMAX [[SEXT_INREG2]], [[SEXT_INREG3]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMAX]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMAX1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SMAX]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SMAX1]], [[C1]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -438,10 +437,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[SMAX:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SMAX]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SMAX1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -566,16 +564,15 @@ body: |
; SI-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR3]], 16
; SI-NEXT: [[SMAX3:%[0-9]+]]:_(s32) = G_SMAX [[SEXT_INREG6]], [[SEXT_INREG7]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMAX]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMAX1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SMAX]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SMAX1]], [[C1]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SMAX2]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SMAX3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SMAX2]], [[C1]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SMAX3]], [[C1]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -609,15 +606,14 @@ body: |
; VI-NEXT: [[SMAX1:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[SMAX2:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[TRUNC3]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SMAX]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SMAX1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SMAX3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SMAX2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SMAX3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
index 7ea14c4693c25..0f7a60198f84b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smin.mir
@@ -413,11 +413,10 @@ body: |
; SI-NEXT: [[SEXT_INREG2:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; SI-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[SMIN1:%[0-9]+]]:_(s32) = G_SMIN [[SEXT_INREG2]], [[SEXT_INREG3]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMIN]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMIN1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SMIN]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SMIN1]], [[C1]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -438,10 +437,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[SMIN:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SMIN]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SMIN1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -566,16 +564,15 @@ body: |
; SI-NEXT: [[SEXT_INREG6:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR1]], 16
; SI-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR3]], 16
; SI-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SEXT_INREG6]], [[SEXT_INREG7]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SMIN]], [[C1]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SMIN1]], [[C1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SMIN]], [[C1]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SMIN1]], [[C1]]
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SMIN2]], [[C1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SMIN3]], [[C1]]
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SMIN2]], [[C1]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SMIN3]], [[C1]]
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -609,15 +606,14 @@ body: |
; VI-NEXT: [[SMIN1:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[SMIN2:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[TRUNC3]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SMIN]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SMIN1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SMIN3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SMIN2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SMIN3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
index 4d84e21a56097..b8817e9e903c0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulh.mir
@@ -169,14 +169,14 @@ body: |
; GFX8-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[UV3]], 16
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG2]], [[SEXT_INREG3]]
; GFX8-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[MUL1]], [[C]](s32)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C1]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C1]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
; GFX8-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX8-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
@@ -338,20 +338,23 @@ body: |
; GFX8-NEXT: [[TRUNC7:%[0-9]+]]:_(s16) = G_TRUNC [[SEXT_INREG7]](s32)
; GFX8-NEXT: [[MUL3:%[0-9]+]]:_(s16) = G_MUL [[TRUNC6]], [[TRUNC7]]
; GFX8-NEXT: [[ASHR3:%[0-9]+]]:_(s16) = G_ASHR [[MUL3]], [[C3]](s16)
- ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; GFX8-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR1]](s16)
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
- ; GFX8-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; GFX8-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR2]](s16)
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
- ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX8-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ASHR3]](s16)
- ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C4]]
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[ASHR]](s16)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C4]]
+ ; GFX8-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[ASHR1]](s16)
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C4]]
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C5]](i32)
+ ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; GFX8-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[ASHR2]](s16)
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ANYEXT2]], [[C4]]
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C6]](i32)
+ ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX8-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[ASHR3]](s16)
+ ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ANYEXT3]], [[C4]]
+ ; GFX8-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX8-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -401,16 +404,18 @@ body: |
; GFX9-NEXT: [[LSHR6:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C4]](i32)
; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[ASHR1]](<2 x s16>)
; GFX9-NEXT: [[LSHR7:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C4]](i32)
- ; GFX9-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C5]]
- ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[LSHR6]], [[C5]]
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
- ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C5]]
- ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
- ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[LSHR7]], [[C5]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX9-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C5]]
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[LSHR6]], [[C5]]
+ ; GFX9-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C6]](i32)
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C5]]
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C4]](i32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[LSHR7]], [[C5]]
+ ; GFX9-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C7]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
index 2fe10bada8dbb..ecab80191db95 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-smulo.mir
@@ -220,11 +220,11 @@ body: |
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG3]], [[SEXT_INREG4]]
; GFX8-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[MUL1]], 16
; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[SEXT_INREG5]]
- ; GFX8-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C]]
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
@@ -233,8 +233,7 @@ body: |
; GFX8-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[ANYEXT1]], 1
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG6]](s32), [[SEXT_INREG7]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C1]](i32)
; GFX8-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[BITCAST1]], 16
; GFX8-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR]], 16
; GFX8-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[SEXT_INREG8]](s32), [[SEXT_INREG9]](s32)
@@ -404,16 +403,19 @@ body: |
; GFX8-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR2]], 8
; GFX8-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR5]], 8
; GFX8-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG7]], [[SEXT_INREG8]]
- ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C3]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C3]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
- ; GFX8-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C3]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
- ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C3]]
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C3]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C3]]
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
+ ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[MUL2]], [[C3]]
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C5]](i32)
+ ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[MUL3]], [[C3]]
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; GFX8-NEXT: $vgpr0 = COPY [[OR2]](s32)
@@ -447,16 +449,19 @@ body: |
; GFX9-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR2]], 8
; GFX9-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR5]], 8
; GFX9-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[SEXT_INREG7]], [[SEXT_INREG8]]
- ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX9-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C3]]
- ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C3]]
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32)
- ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C3]]
- ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
- ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C3]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX9-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C3]]
+ ; GFX9-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C3]]
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C4]](i32)
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; GFX9-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[MUL2]], [[C3]]
+ ; GFX9-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C5]](i32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX9-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[MUL3]], [[C3]]
+ ; GFX9-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C6]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
index 489180a3b986d..63ba0a935a785 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-srem.mir
@@ -2656,11 +2656,10 @@ body: |
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
; GFX6-NEXT: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
; GFX6-NEXT: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C4]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB9]], [[C4]]
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB4]], [[C4]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB9]], [[C4]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2732,11 +2731,10 @@ body: |
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB8]], [[SELECT2]]
; GFX8-NEXT: [[XOR5:%[0-9]+]]:_(s32) = G_XOR [[SELECT3]], [[ASHR2]]
; GFX8-NEXT: [[SUB9:%[0-9]+]]:_(s32) = G_SUB [[XOR5]], [[ASHR2]]
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX8-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB4]], [[C4]]
- ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB9]], [[C4]]
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C5]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB4]], [[C4]]
+ ; GFX8-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB9]], [[C4]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir
index 01a72aef58642..7ffee124d3391 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sshlsat.mir
@@ -425,9 +425,10 @@ body: |
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[ASHR2]]
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SELECT2]], [[SHL3]]
; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SELECT3]], [[C2]](s32)
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C1]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C6]]
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR3]], [[C6]]
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -461,10 +462,9 @@ body: |
; GFX8-NEXT: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP2]](s1), [[C1]], [[C2]]
; GFX8-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC1]](s16), [[ASHR1]]
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP3]](s1), [[SELECT2]], [[SHL1]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT3]](s16)
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -554,16 +554,18 @@ body: |
; GFX6-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL4]](s32), [[ASHR4]]
; GFX6-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP5]](s1), [[SELECT4]], [[SHL5]]
; GFX6-NEXT: [[ASHR5:%[0-9]+]]:_(s32) = G_ASHR [[SELECT5]], [[C1]](s32)
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C5]]
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C6]]
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR3]], [[C6]]
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR5]], [[C5]]
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ASHR5]], [[C6]]
+ ; GFX6-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C7]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL7]]
+ ; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C7]], [[SHL7]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX6-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -607,18 +609,17 @@ body: |
; GFX8-NEXT: [[SELECT4:%[0-9]+]]:_(s16) = G_SELECT [[ICMP4]](s1), [[C1]], [[C2]]
; GFX8-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC2]](s16), [[ASHR2]]
; GFX8-NEXT: [[SELECT5:%[0-9]+]]:_(s16) = G_SELECT [[ICMP5]](s1), [[SELECT4]], [[SHL2]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT3]](s16)
+ ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT5]](s16)
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT5]](s16)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL4]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL4]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -737,14 +738,15 @@ body: |
; GFX6-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL6]](s32), [[ASHR6]]
; GFX6-NEXT: [[SELECT7:%[0-9]+]]:_(s32) = G_SELECT [[ICMP7]](s1), [[SELECT6]], [[SHL7]]
; GFX6-NEXT: [[ASHR7:%[0-9]+]]:_(s32) = G_ASHR [[SELECT7]], [[C2]](s32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C1]]
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C6]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ASHR3]], [[C6]]
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ASHR5]], [[C1]]
- ; GFX6-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ASHR7]], [[C1]]
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND5]], [[C2]](s32)
+ ; GFX6-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ASHR5]], [[C6]]
+ ; GFX6-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[ASHR7]], [[C6]]
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -801,15 +803,14 @@ body: |
; GFX8-NEXT: [[SELECT6:%[0-9]+]]:_(s16) = G_SELECT [[ICMP6]](s1), [[C1]], [[C2]]
; GFX8-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC3]](s16), [[ASHR3]]
; GFX8-NEXT: [[SELECT7:%[0-9]+]]:_(s16) = G_SELECT [[ICMP7]](s1), [[SELECT6]], [[SHL3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT3]](s16)
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT5]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT7]](s16)
- ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT5]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT7]](s16)
+ ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir
index ba9e7a971cde6..c5f29a09edef7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubo.mir
@@ -147,11 +147,10 @@ body: |
; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST1]]
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR1]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(i32) = G_BITCAST [[COPY]](<2 x s16>)
@@ -165,19 +164,19 @@ body: |
; CHECK-NEXT: [[SEXT_INREG3:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR3]], 16
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG2]](s32), [[SEXT_INREG3]]
; CHECK-NEXT: [[SEXT_INREG4:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB]], 16
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG4]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG5:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB1]], 16
- ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C3]]
+ ; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG5]](s32), [[C2]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[ICMP2]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP3]]
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1)
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(<2 x s16>) = COPY [[BITCAST2]](<2 x s16>)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND2]](s32), [[AND3]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[COPY3]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
@@ -243,19 +242,19 @@ body: |
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR1]](s1)
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C2]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C2]]
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C2]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C2]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SUB2]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C1]], [[C3]](s32)
- ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL2]]
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
+ ; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL2]]
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>), [[BITCAST10]](<2 x s16>)
; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
@@ -303,16 +302,15 @@ body: |
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR2]]
; CHECK-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[BITCAST1]], [[BITCAST3]]
; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[LSHR1]], [[LSHR3]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C1]]
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C1]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C1]]
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C1]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SUB2]], [[C1]]
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SUB3]], [[C1]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -339,17 +337,17 @@ body: |
; CHECK-NEXT: [[SEXT_INREG7:%[0-9]+]]:_(s32) = G_SEXT_INREG [[LSHR7]], 16
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG6]](s32), [[SEXT_INREG7]]
; CHECK-NEXT: [[SEXT_INREG8:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB]], 16
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP4:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG8]](s32), [[COPY2]]
; CHECK-NEXT: [[SEXT_INREG9:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB1]], 16
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP5:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG9]](s32), [[COPY3]]
; CHECK-NEXT: [[SEXT_INREG10:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB2]], 16
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C3]](s32)
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[C2]](s32)
; CHECK-NEXT: [[ICMP6:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG10]](s32), [[COPY4]]
; CHECK-NEXT: [[SEXT_INREG11:%[0-9]+]]:_(s32) = G_SEXT_INREG [[SUB3]], 16
- ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C3]]
+ ; CHECK-NEXT: [[ICMP7:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[SEXT_INREG11]](s32), [[C2]]
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[ICMP4]]
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[ICMP1]], [[ICMP5]]
; CHECK-NEXT: [[XOR2:%[0-9]+]]:_(s1) = G_XOR [[ICMP2]], [[ICMP6]]
@@ -359,11 +357,11 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[XOR3]](s1)
; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(<4 x s16>) = COPY [[CONCAT_VECTORS]](<4 x s16>)
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
- ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C4]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32), [[AND6]](s32), [[AND7]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[COPY5]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
index afb674a064877..91aadd3c8700f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ssubsat.mir
@@ -397,10 +397,10 @@ body: |
; GFX6-NEXT: [[SMIN3:%[0-9]+]]:_(s32) = G_SMIN [[SMAX3]], [[SUB4]]
; GFX6-NEXT: [[SUB5:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[SMIN3]]
; GFX6-NEXT: [[ASHR1:%[0-9]+]]:_(s32) = G_ASHR [[SUB5]], [[C1]](s32)
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -436,10 +436,9 @@ body: |
; GFX8-NEXT: [[SMAX3:%[0-9]+]]:_(s16) = G_SMAX [[SUB3]], [[TRUNC3]]
; GFX8-NEXT: [[SMIN3:%[0-9]+]]:_(s16) = G_SMIN [[SMAX3]], [[SUB4]]
; GFX8-NEXT: [[SUB5:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[SMIN3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SUB2]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SUB5]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -509,15 +508,15 @@ body: |
; GFX6-NEXT: [[SMIN5:%[0-9]+]]:_(s32) = G_SMIN [[SMAX5]], [[SUB7]]
; GFX6-NEXT: [[SUB8:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[SMIN5]]
; GFX6-NEXT: [[ASHR2:%[0-9]+]]:_(s32) = G_ASHR [[SUB8]], [[C1]](s32)
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
- ; GFX6-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C6]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C6]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C6]], [[SHL7]]
@@ -567,18 +566,17 @@ body: |
; GFX8-NEXT: [[SMAX5:%[0-9]+]]:_(s16) = G_SMAX [[SUB6]], [[TRUNC5]]
; GFX8-NEXT: [[SMIN5:%[0-9]+]]:_(s16) = G_SMIN [[SMAX5]], [[SUB7]]
; GFX8-NEXT: [[SUB8:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[SMIN5]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SUB2]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SUB5]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB8]](s16)
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C5]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SUB8]](s16)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C5]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -686,15 +684,15 @@ body: |
; GFX6-NEXT: [[SMIN7:%[0-9]+]]:_(s32) = G_SMIN [[SMAX7]], [[SUB10]]
; GFX6-NEXT: [[SUB11:%[0-9]+]]:_(s32) = G_SUB [[SHL6]], [[SMIN7]]
; GFX6-NEXT: [[ASHR3:%[0-9]+]]:_(s32) = G_ASHR [[SUB11]], [[C1]](s32)
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ASHR]], [[C5]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[ASHR1]], [[C5]]
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[ASHR]], [[C5]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[ASHR1]], [[C5]]
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[ASHR2]], [[C5]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ASHR3]], [[C5]]
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[ASHR2]], [[C5]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[ASHR3]], [[C5]]
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -755,15 +753,14 @@ body: |
; GFX8-NEXT: [[SMAX7:%[0-9]+]]:_(s16) = G_SMAX [[SUB9]], [[TRUNC7]]
; GFX8-NEXT: [[SMIN7:%[0-9]+]]:_(s16) = G_SMIN [[SMAX7]], [[SUB10]]
; GFX8-NEXT: [[SUB11:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[SMIN7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB5]](s16)
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SUB2]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SUB5]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB8]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB11]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C4]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SUB8]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SUB11]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
index 0c4c3de3ffaf7..17fe6f73a8188 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store-global.mir
@@ -8424,18 +8424,18 @@ body: |
; SI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
; SI-NEXT: {{ $}}
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; SI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; SI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; SI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; SI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; SI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; SI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -8456,18 +8456,18 @@ body: |
; CI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
; CI-NEXT: {{ $}}
; CI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
- ; CI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; CI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; CI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; CI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
@@ -8488,18 +8488,18 @@ body: |
; VI: liveins: $vgpr0_vgpr1, $vgpr2_vgpr3_vgpr4_vgpr5_vgpr6
; VI-NEXT: {{ $}}
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; VI-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; VI-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; VI-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>), [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
index 6cca206efe4a4..a951a9f359812 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-store.mir
@@ -814,22 +814,21 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; SI-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[OR2]](s32)
- ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C4]](i32)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
- ; SI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C7]]
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[AND4]], [[C6]](i32)
- ; SI-NEXT: [[C8:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C8]](i64)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C3]](i32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY3]], [[C6]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[AND4]], [[C5]](i32)
+ ; SI-NEXT: [[C7:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C7]](i64)
; SI-NEXT: G_STORE [[COPY3]](i32), [[COPY]](p1) :: (store (s8), addrspace 1)
; SI-NEXT: G_STORE [[LSHR1]](i32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
@@ -855,20 +854,19 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](s32)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C3]](i32)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
- ; VI-NEXT: [[C5:%[0-9]+]]:_(i16) = G_CONSTANT i16 8
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i16) = G_LSHR [[OR]], [[C5]](i16)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i16) = G_CONSTANT i16 8
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i16) = G_LSHR [[OR]], [[C4]](i16)
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
; VI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s8), addrspace 1)
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR1]](i16)
; VI-NEXT: G_STORE [[ANYEXT]](i32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
@@ -909,16 +907,15 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; SI-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[OR2]](s32)
- ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C4]](i32)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C3]](i32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; SI-NEXT: G_STORE [[COPY3]](i32), [[COPY]](p1) :: (store (s16), addrspace 1)
; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
;
@@ -943,16 +940,15 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](s32)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C3]](i32)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s16), addrspace 1)
; VI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
%0:_(p1) = COPY $vgpr0_vgpr1
@@ -991,16 +987,15 @@ body: |
; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
; SI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[SHL1]](s32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[TRUNC3]]
- ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; SI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; SI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; SI-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[OR2]](s32)
- ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C4]](i32)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY3]], [[C3]](i32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; SI-NEXT: G_STORE [[COPY3]](i32), [[COPY]](p1) :: (store (s16), align 4, addrspace 1)
; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
;
@@ -1025,16 +1020,15 @@ body: |
; VI-NEXT: [[AND3:%[0-9]+]]:_(s16) = G_AND [[TRUNC3]], [[C]]
; VI-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND3]], [[C1]](s16)
; VI-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[AND2]], [[SHL1]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; VI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](s32)
- ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C3]](i32)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C3]](i64)
; VI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s16), align 4, addrspace 1)
; VI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, align 2, addrspace 1)
%0:_(p1) = COPY $vgpr0_vgpr1
@@ -1056,36 +1050,34 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
; SI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](i32)
- ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C4]](i32)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
- ; SI-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
- ; SI-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
- ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C7]]
- ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[AND4]], [[C6]](i32)
- ; SI-NEXT: [[C8:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C8]](i64)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
+ ; SI-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; SI-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[COPY2]], [[C5]]
+ ; SI-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[AND4]], [[C1]](i32)
+ ; SI-NEXT: [[C6:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; SI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C6]](i64)
; SI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s8), addrspace 1)
; SI-NEXT: G_STORE [[LSHR1]](i32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
- ; SI-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[C6]](i32)
+ ; SI-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[C1]](i32)
; SI-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[LSHR]], [[COPY3]](i32)
- ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C8]](i64)
+ ; SI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C6]](i64)
; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
; SI-NEXT: G_STORE [[LSHR2]](i32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
;
@@ -1095,43 +1087,42 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
; VI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](i32)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C4]](i32)
- ; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: [[TRUNC:%[0-9]+]]:_(s16) = G_TRUNC [[UV]](s32)
- ; VI-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
- ; VI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C6]]
+ ; VI-NEXT: [[C5:%[0-9]+]]:_(s16) = G_CONSTANT i16 255
+ ; VI-NEXT: [[AND4:%[0-9]+]]:_(s16) = G_AND [[TRUNC]], [[C5]]
; VI-NEXT: [[TRUNC1:%[0-9]+]]:_(s16) = G_TRUNC [[UV1]](s32)
- ; VI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C6]]
- ; VI-NEXT: [[C7:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
- ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C7]](s16)
+ ; VI-NEXT: [[AND5:%[0-9]+]]:_(s16) = G_AND [[TRUNC1]], [[C5]]
+ ; VI-NEXT: [[C6:%[0-9]+]]:_(s16) = G_CONSTANT i16 8
+ ; VI-NEXT: [[SHL3:%[0-9]+]]:_(s16) = G_SHL [[AND5]], [[C6]](s16)
; VI-NEXT: [[OR3:%[0-9]+]]:_(i16) = G_OR [[AND4]], [[SHL3]]
- ; VI-NEXT: [[C8:%[0-9]+]]:_(i16) = G_CONSTANT i16 8
- ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i16) = G_LSHR [[OR3]], [[C8]](i16)
- ; VI-NEXT: [[C9:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
- ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C9]](i64)
+ ; VI-NEXT: [[C7:%[0-9]+]]:_(i16) = G_CONSTANT i16 8
+ ; VI-NEXT: [[LSHR1:%[0-9]+]]:_(i16) = G_LSHR [[OR3]], [[C7]](i16)
+ ; VI-NEXT: [[C8:%[0-9]+]]:_(i64) = G_CONSTANT i64 1
+ ; VI-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C8]](i64)
; VI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s8), addrspace 1)
; VI-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR1]](i16)
; VI-NEXT: G_STORE [[ANYEXT]](i32), [[PTR_ADD1]](p1) :: (store (s8) into unknown-address + 1, addrspace 1)
; VI-NEXT: [[TRUNC2:%[0-9]+]]:_(i16) = G_TRUNC [[LSHR]](i32)
- ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i16) = G_LSHR [[TRUNC2]], [[C8]](i16)
- ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C9]](i64)
+ ; VI-NEXT: [[LSHR2:%[0-9]+]]:_(i16) = G_LSHR [[TRUNC2]], [[C7]](i16)
+ ; VI-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[PTR_ADD]], [[C8]](i64)
; VI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s8) into unknown-address + 2, addrspace 1)
; VI-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR2]](i16)
; VI-NEXT: G_STORE [[ANYEXT1]](i32), [[PTR_ADD2]](p1) :: (store (s8) into unknown-address + 3, addrspace 1)
@@ -1154,25 +1145,24 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
; SI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](i32)
- ; SI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C4]](i32)
- ; SI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
+ ; SI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; SI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; SI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; SI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s16), addrspace 1)
; SI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
;
@@ -1182,25 +1172,24 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
; VI-NEXT: [[COPY2:%[0-9]+]]:_(i32) = COPY [[OR2]](i32)
- ; VI-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C4]](i32)
- ; VI-NEXT: [[C5:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
- ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C5]](i64)
+ ; VI-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[COPY2]], [[C2]](i32)
+ ; VI-NEXT: [[C4:%[0-9]+]]:_(i64) = G_CONSTANT i64 2
+ ; VI-NEXT: [[PTR_ADD:%[0-9]+]]:_(p1) = nuw inbounds G_PTR_ADD [[COPY]], [[C4]](i64)
; VI-NEXT: G_STORE [[COPY2]](i32), [[COPY]](p1) :: (store (s16), addrspace 1)
; VI-NEXT: G_STORE [[LSHR]](i32), [[PTR_ADD]](p1) :: (store (s16) into unknown-address + 2, addrspace 1)
%0:_(p1) = COPY $vgpr0_vgpr1
@@ -1222,19 +1211,19 @@ body: |
; SI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; SI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
; SI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; SI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; SI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; SI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; SI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; SI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; SI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; SI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; SI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; SI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; SI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; SI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; SI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; SI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; SI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; SI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; SI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; SI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; SI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
; SI-NEXT: G_STORE [[OR2]](i32), [[COPY]](p1) :: (store (i32), addrspace 1)
;
@@ -1244,19 +1233,19 @@ body: |
; VI-NEXT: [[COPY:%[0-9]+]]:_(p1) = COPY $vgpr0_vgpr1
; VI-NEXT: [[COPY1:%[0-9]+]]:_(<4 x s32>) = COPY $vgpr2_vgpr3_vgpr4_vgpr5
; VI-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32), [[UV2:%[0-9]+]]:_(s32), [[UV3:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[COPY1]](<4 x s32>)
- ; VI-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; VI-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[UV]], [[C]]
- ; VI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[UV1]], [[C]]
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 8
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
- ; VI-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]]
- ; VI-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C]]
- ; VI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C2]](s32)
- ; VI-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; VI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
- ; VI-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 24
- ; VI-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C3]](s32)
+ ; VI-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; VI-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[UV]], [[C]]
+ ; VI-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[UV1]], [[C]]
+ ; VI-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
+ ; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
+ ; VI-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C]]
+ ; VI-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C2]](i32)
+ ; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; VI-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV3]], [[C]]
+ ; VI-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; VI-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C3]](i32)
; VI-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[OR1]], [[SHL2]]
; VI-NEXT: G_STORE [[OR2]](i32), [[COPY]](p1) :: (store (i32), addrspace 1)
%0:_(p1) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
index 12d0bf24305d2..dc4c169d008f1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-sub.mir
@@ -151,11 +151,10 @@ body: |
; GFX6-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C]](i32)
; GFX6-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[BITCAST]], [[BITCAST1]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR1]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -176,10 +175,9 @@ body: |
; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[SUB:%[0-9]+]]:_(s16) = G_SUB [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[TRUNC3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SUB]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SUB1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -314,16 +312,15 @@ body: |
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR2]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[BITCAST1]], [[BITCAST3]]
; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[LSHR1]], [[LSHR3]]
- ; GFX6-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; GFX6-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[SUB]], [[C1]]
- ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C2]](s32)
+ ; GFX6-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C1]]
+ ; GFX6-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C1]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SUB2]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C1]]
- ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C2]](s32)
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SUB2]], [[C1]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SUB3]], [[C1]]
+ ; GFX6-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -357,15 +354,14 @@ body: |
; GFX8-NEXT: [[SUB1:%[0-9]+]]:_(s16) = G_SUB [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[SUB2:%[0-9]+]]:_(s16) = G_SUB [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[SUB3:%[0-9]+]]:_(s16) = G_SUB [[TRUNC3]], [[TRUNC7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SUB]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SUB1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SUB]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SUB1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SUB2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SUB3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SUB2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SUB3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trunc.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trunc.mir
index 0711c47db0a96..cb8cb46798fcc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trunc.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-trunc.mir
@@ -126,13 +126,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<2 x s64>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[UV]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[UV1]](s64)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[UV]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i32) = G_TRUNC [[UV1]](s64)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -152,20 +152,20 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<4 x s64>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64), [[UV3:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY]](<4 x s64>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[UV]](s64)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[UV1]](s64)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[UV]](s64)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i32) = G_TRUNC [[UV1]](s64)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(s32) = G_TRUNC [[UV2]](s64)
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[TRUNC2]], [[C]]
- ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(s32) = G_TRUNC [[UV3]](s64)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[TRUNC3]], [[C]]
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC2:%[0-9]+]]:_(i32) = G_TRUNC [[UV2]](s64)
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[TRUNC2]], [[C]]
+ ; CHECK-NEXT: [[TRUNC3:%[0-9]+]]:_(i32) = G_TRUNC [[UV3]](s64)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[TRUNC3]], [[C]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
@@ -353,13 +353,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(<2 x s128>) = COPY $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
; CHECK-NEXT: [[UV:%[0-9]+]]:_(s128), [[UV1:%[0-9]+]]:_(s128) = G_UNMERGE_VALUES [[COPY]](<2 x s128>)
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[UV]](s128)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[UV1]](s128)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[UV]](s128)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i32) = G_TRUNC [[UV1]](s128)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -401,13 +401,13 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s96) = COPY $vgpr0_vgpr1_vgpr2
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s96) = COPY $vgpr3_vgpr4_vgpr5
- ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s96)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]]
- ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(s32) = G_TRUNC [[COPY1]](s96)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[TRUNC1]], [[C]]
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C1]](s32)
+ ; CHECK-NEXT: [[TRUNC:%[0-9]+]]:_(i32) = G_TRUNC [[COPY]](s96)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[TRUNC]], [[C]]
+ ; CHECK-NEXT: [[TRUNC1:%[0-9]+]]:_(i32) = G_TRUNC [[COPY1]](s96)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[TRUNC1]], [[C]]
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND1]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
index e3a3029f54d4d..fc0d2d9d29887 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddo.mir
@@ -142,18 +142,18 @@ body: |
; CHECK-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[ADD1]], [[C1]]
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD1]](s32), [[AND3]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C2]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32)
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND6]](s32), [[AND7]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -200,25 +200,25 @@ body: |
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C2]]
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND8]], [[C]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND7]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND9]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND7]](s32), [[AND8]](s32), [[AND9]](s32)
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND10]](s32), [[AND11]](s32), [[AND12]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
%0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
@@ -272,16 +272,16 @@ body: |
; CHECK-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[LSHR1]], [[LSHR3]]
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ADD3]], [[C1]]
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD3]](s32), [[AND7]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[ADD]], [[C2]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[ADD1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[AND7]](s32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[ADD2]], [[C2]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[ADD3]], [[C2]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
@@ -289,11 +289,11 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP3]](s1)
; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32), [[AND10]](s32), [[AND11]](s32)
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND12]](s32), [[AND13]](s32), [[AND14]](s32), [[AND15]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
index b80b811b2a969..3478140c68aa7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uaddsat.mir
@@ -310,7 +310,7 @@ body: |
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[XOR1]], [[SHL3]]
; GFX6-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[SHL2]], [[UMIN1]]
; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[ADD1]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR2]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -331,10 +331,9 @@ body: |
; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -390,11 +389,11 @@ body: |
; GFX6-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[XOR2]], [[SHL5]]
; GFX6-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[SHL4]], [[UMIN2]]
; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[ADD2]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR4]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR4]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C1]](s32)
+ ; GFX6-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL7]]
@@ -423,18 +422,17 @@ body: |
; GFX8-NEXT: [[UADDSAT:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC]], [[TRUNC3]]
; GFX8-NEXT: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC4]]
; GFX8-NEXT: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC2]], [[TRUNC5]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT2]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT2]](s16)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -524,10 +522,10 @@ body: |
; GFX6-NEXT: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[XOR3]], [[SHL7]]
; GFX6-NEXT: [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[SHL6]], [[UMIN3]]
; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[ADD3]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR5]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR6]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -561,15 +559,14 @@ body: |
; GFX8-NEXT: [[UADDSAT1:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[UADDSAT2:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[UADDSAT3:%[0-9]+]]:_(s16) = G_UADDSAT [[TRUNC3]], [[TRUNC7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UADDSAT3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[UADDSAT3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
index 6a8e16aeec0d0..ab184c3e94a59 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-udiv.mir
@@ -2343,10 +2343,10 @@ body: |
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](i32), [[LSHR1]]
; GFX6-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[SELECT3]], [[C4]]
; GFX6-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT2]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT5]], [[C1]]
- ; GFX6-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; GFX6-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SELECT2]], [[C5]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SELECT5]], [[C5]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2405,10 +2405,10 @@ body: |
; GFX8-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT4]](i32), [[LSHR1]]
; GFX8-NEXT: [[ADD5:%[0-9]+]]:_(i32) = G_ADD [[SELECT3]], [[C4]]
; GFX8-NEXT: [[SELECT5:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[ADD5]], [[SELECT3]]
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT2]], [[C1]]
- ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT5]], [[C1]]
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C5]](s32)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SELECT2]], [[C5]]
+ ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SELECT5]], [[C5]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
index 9bca5fc90dc3b..2f8dd70b1b88e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-uitofp.mir
@@ -575,10 +575,10 @@ body: |
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[C]], [[UMIN2]]
; GFX6-NEXT: [[FLDEXP1:%[0-9]+]]:_(s32) = G_FLDEXP [[UITOFP1]], [[SUB1]](i32)
; GFX6-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FLDEXP1]](s32)
- ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX6-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX6-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX6-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX6-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
@@ -612,10 +612,10 @@ body: |
; GFX8-NEXT: [[SUB1:%[0-9]+]]:_(i32) = G_SUB [[C]], [[UMIN2]]
; GFX8-NEXT: [[FLDEXP1:%[0-9]+]]:_(s32) = G_FLDEXP [[UITOFP1]], [[SUB1]](i32)
; GFX8-NEXT: [[FPTRUNC1:%[0-9]+]]:_(s16) = G_FPTRUNC [[FLDEXP1]](s32)
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FPTRUNC1]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FPTRUNC1]](s16)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
index 6521bfe705daa..fbe47638d57e5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umax.mir
@@ -421,8 +421,7 @@ body: |
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[UMAX:%[0-9]+]]:_(s32) = G_UMAX [[AND]], [[AND1]]
; SI-NEXT: [[UMAX1:%[0-9]+]]:_(s32) = G_UMAX [[LSHR]], [[LSHR1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMAX1]], [[C2]](s32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[UMAX1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMAX]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -443,10 +442,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UMAX:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[UMAX1:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UMAX]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UMAX1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -568,11 +566,10 @@ body: |
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[UMAX2:%[0-9]+]]:_(s32) = G_UMAX [[AND2]], [[AND3]]
; SI-NEXT: [[UMAX3:%[0-9]+]]:_(s32) = G_UMAX [[LSHR1]], [[LSHR3]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMAX1]], [[C2]](s32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[UMAX1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMAX]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UMAX3]], [[C2]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[UMAX3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[UMAX2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -606,15 +603,14 @@ body: |
; VI-NEXT: [[UMAX1:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[UMAX2:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[UMAX3:%[0-9]+]]:_(s16) = G_UMAX [[TRUNC3]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UMAX]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UMAX1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UMAX3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UMAX2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[UMAX3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
index 64584d0b9a425..021e84bb93d79 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umin.mir
@@ -421,8 +421,7 @@ body: |
; SI-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C1]]
; SI-NEXT: [[UMIN:%[0-9]+]]:_(s32) = G_UMIN [[AND]], [[AND1]]
; SI-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[LSHR]], [[LSHR1]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMIN1]], [[C2]](s32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[UMIN1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMIN]], [[SHL]]
; SI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; SI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -443,10 +442,9 @@ body: |
; VI-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; VI-NEXT: [[UMIN:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC]], [[TRUNC2]]
; VI-NEXT: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC3]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UMIN]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UMIN1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; VI-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -568,11 +566,10 @@ body: |
; SI-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[BITCAST3]], [[C1]]
; SI-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[AND2]], [[AND3]]
; SI-NEXT: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[LSHR1]], [[LSHR3]]
- ; SI-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; SI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[UMIN1]], [[C2]](s32)
+ ; SI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[UMIN1]], [[C]](i32)
; SI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[UMIN]], [[SHL]]
; SI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; SI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[UMIN3]], [[C2]](s32)
+ ; SI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[UMIN3]], [[C]](i32)
; SI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[UMIN2]], [[SHL1]]
; SI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; SI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -606,15 +603,14 @@ body: |
; VI-NEXT: [[UMIN1:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC1]], [[TRUNC5]]
; VI-NEXT: [[UMIN2:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC2]], [[TRUNC6]]
; VI-NEXT: [[UMIN3:%[0-9]+]]:_(s16) = G_UMIN [[TRUNC3]], [[TRUNC7]]
- ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN]](s16)
- ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN1]](s16)
- ; VI-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; VI-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[UMIN]](s16)
+ ; VI-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[UMIN1]](s16)
+ ; VI-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; VI-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; VI-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN2]](s16)
- ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[UMIN3]](s16)
- ; VI-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; VI-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[UMIN2]](s16)
+ ; VI-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[UMIN3]](s16)
+ ; VI-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; VI-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; VI-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; VI-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
index 6679247346de6..f08dbcaafcdcb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulh.mir
@@ -376,11 +376,11 @@ body: |
; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV3]], [[C]]
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND2]], [[AND3]]
; GFX8-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[MUL1]], [[C1]](s32)
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C1]](s32)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
; GFX8-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
; GFX8-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C]]
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[LSHR2]](i32)
@@ -455,10 +455,10 @@ body: |
; GFX8-NEXT: [[AND6:%[0-9]+]]:_(s16) = G_AND [[TRUNC6]], [[C]]
; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND6]], [[C1]](s16)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[LSHR2]], [[SHL1]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -506,10 +506,9 @@ body: |
; GFX9-NEXT: [[AND8:%[0-9]+]]:_(s16) = G_AND [[TRUNC8]], [[C]]
; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[AND8]], [[C1]](s16)
; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s16) = G_OR [[LSHR1]], [[SHL1]]
- ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](s16)
- ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s16)
- ; GFX9-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C3]](s32)
+ ; GFX9-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](s16)
+ ; GFX9-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](s16)
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C2]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(s32) = COPY $vgpr0
@@ -659,20 +658,23 @@ body: |
; GFX8-NEXT: [[AND7:%[0-9]+]]:_(s16) = G_AND [[TRUNC7]], [[C3]]
; GFX8-NEXT: [[MUL3:%[0-9]+]]:_(s16) = G_MUL [[AND6]], [[AND7]]
; GFX8-NEXT: [[LSHR9:%[0-9]+]]:_(s16) = G_LSHR [[MUL3]], [[C4]](s16)
- ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR6]](s16)
- ; GFX8-NEXT: [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX8-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C5]]
- ; GFX8-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR7]](s16)
- ; GFX8-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C5]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C]](s32)
- ; GFX8-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL]]
- ; GFX8-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR8]](s16)
- ; GFX8-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C5]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C1]](s32)
- ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX8-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[LSHR9]](s16)
- ; GFX8-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C5]]
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C2]](s32)
+ ; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR6]](s16)
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX8-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[ANYEXT]], [[C5]]
+ ; GFX8-NEXT: [[ANYEXT1:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR7]](s16)
+ ; GFX8-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[ANYEXT1]], [[C5]]
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C6]](i32)
+ ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL]]
+ ; GFX8-NEXT: [[ANYEXT2:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR8]](s16)
+ ; GFX8-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[ANYEXT2]], [[C5]]
+ ; GFX8-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C7]](i32)
+ ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX8-NEXT: [[ANYEXT3:%[0-9]+]]:_(i32) = G_ANYEXT [[LSHR9]](s16)
+ ; GFX8-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[ANYEXT3]], [[C5]]
+ ; GFX8-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C8]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX8-NEXT: $vgpr0 = COPY [[OR2]](s32)
;
@@ -723,16 +725,18 @@ body: |
; GFX9-NEXT: [[LSHR8:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST]], [[C5]](i32)
; GFX9-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[LSHR7]](<2 x s16>)
; GFX9-NEXT: [[LSHR9:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C5]](i32)
- ; GFX9-NEXT: [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 255
- ; GFX9-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C6]]
- ; GFX9-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[LSHR8]], [[C6]]
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C]](s32)
- ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND8]], [[SHL]]
- ; GFX9-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C6]]
- ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C1]](s32)
- ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX9-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[LSHR9]], [[C6]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND11]], [[C2]](s32)
+ ; GFX9-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[BITCAST]], [[C6]]
+ ; GFX9-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[LSHR8]], [[C6]]
+ ; GFX9-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C7]](i32)
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL]]
+ ; GFX9-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[BITCAST1]], [[C6]]
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C5]](i32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX9-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[LSHR9]], [[C6]]
+ ; GFX9-NEXT: [[C8:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C8]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
%0:_(s32) = COPY $vgpr0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
index c61932783bebd..5ded894c166d7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-umulo.mir
@@ -568,21 +568,21 @@ body: |
; GFX8-NEXT: [[MUL1:%[0-9]+]]:_(s32) = G_MUL [[AND3]], [[AND4]]
; GFX8-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[MUL1]](s32), [[AND5]]
- ; GFX8-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C]]
- ; GFX8-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C]]
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND7]], [[C1]](s32)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND6:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C1]]
+ ; GFX8-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C1]]
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND7]], [[C2]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND6]], [[SHL]]
; GFX8-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; GFX8-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; GFX8-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C2]]
- ; GFX8-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C2]]
+ ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+ ; GFX8-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; GFX8-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
; GFX8-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32)
; GFX8-NEXT: [[BITCAST1:%[0-9]+]]:_(i32) = G_BITCAST [[BITCAST]](<2 x s16>)
- ; GFX8-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C3]](i32)
+ ; GFX8-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST1]], [[C2]](i32)
; GFX8-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[BITCAST1]], [[C]]
; GFX8-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND10]](s32), [[LSHR]](i32)
; GFX8-NEXT: $vgpr0_vgpr1 = COPY [[BUILD_VECTOR1]](<2 x s32>)
@@ -754,15 +754,19 @@ body: |
; GFX8-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C3]]
; GFX8-NEXT: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[AND6]]
; GFX8-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[LSHR2]], [[LSHR5]]
- ; GFX8-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C3]]
- ; GFX8-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C3]]
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C]](s32)
- ; GFX8-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND7]], [[SHL]]
- ; GFX8-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C3]]
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
- ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX8-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C3]]
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C2]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX8-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C4]]
+ ; GFX8-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C4]]
+ ; GFX8-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND8]], [[C5]](i32)
+ ; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND7]], [[SHL]]
+ ; GFX8-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[MUL2]], [[C4]]
+ ; GFX8-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C6]](i32)
+ ; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX8-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[MUL3]], [[C4]]
+ ; GFX8-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C7]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX8-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; GFX8-NEXT: $vgpr0 = COPY [[OR2]](s32)
@@ -795,15 +799,19 @@ body: |
; GFX9-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[LSHR4]], [[C3]]
; GFX9-NEXT: [[MUL2:%[0-9]+]]:_(s32) = G_MUL [[AND5]], [[AND6]]
; GFX9-NEXT: [[MUL3:%[0-9]+]]:_(s32) = G_MUL [[LSHR2]], [[LSHR5]]
- ; GFX9-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[MUL]], [[C3]]
- ; GFX9-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[MUL1]], [[C3]]
- ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND8]], [[C]](s32)
- ; GFX9-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND7]], [[SHL]]
- ; GFX9-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[MUL2]], [[C3]]
- ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[AND9]], [[C1]](s32)
- ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GFX9-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[MUL3]], [[C3]]
- ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[AND10]], [[C2]](s32)
+ ; GFX9-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 255
+ ; GFX9-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[MUL]], [[C4]]
+ ; GFX9-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[MUL1]], [[C4]]
+ ; GFX9-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 8
+ ; GFX9-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND8]], [[C5]](i32)
+ ; GFX9-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND7]], [[SHL]]
+ ; GFX9-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[MUL2]], [[C4]]
+ ; GFX9-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; GFX9-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C6]](i32)
+ ; GFX9-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[OR]], [[SHL1]]
+ ; GFX9-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[MUL3]], [[C4]]
+ ; GFX9-NEXT: [[C7:%[0-9]+]]:_(i32) = G_CONSTANT i32 24
+ ; GFX9-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[AND10]], [[C7]](i32)
; GFX9-NEXT: [[OR2:%[0-9]+]]:_(s32) = G_OR [[OR1]], [[SHL2]]
; GFX9-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; GFX9-NEXT: $vgpr0 = COPY [[OR2]](s32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
index f7ffba8c53491..a7ecfe3d7c3d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-urem.mir
@@ -2250,10 +2250,10 @@ body: |
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](i32), [[LSHR1]]
; GFX6-NEXT: [[SUB7:%[0-9]+]]:_(i32) = G_SUB [[SELECT2]], [[LSHR1]]
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
- ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C1]]
- ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT3]], [[C1]]
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX6-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SELECT1]], [[C4]]
+ ; GFX6-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SELECT3]], [[C4]]
+ ; GFX6-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -2307,10 +2307,10 @@ body: |
; GFX8-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[SELECT2]](i32), [[LSHR1]]
; GFX8-NEXT: [[SUB7:%[0-9]+]]:_(i32) = G_SUB [[SELECT2]], [[LSHR1]]
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[SUB7]], [[SELECT2]]
- ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[SELECT1]], [[C1]]
- ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SELECT3]], [[C1]]
- ; GFX8-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND3]], [[C4]](s32)
+ ; GFX8-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; GFX8-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[SELECT1]], [[C4]]
+ ; GFX8-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[SELECT3]], [[C4]]
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND3]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND2]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir
index 008601ba9564e..2df2b7e012ecb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ushlsat.mir
@@ -358,7 +358,7 @@ body: |
; GFX6-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL2]](s32), [[LSHR4]]
; GFX6-NEXT: [[SELECT1:%[0-9]+]]:_(s32) = G_SELECT [[ICMP1]](s1), [[C3]], [[SHL3]]
; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SELECT1]], [[C2]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR5]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -386,10 +386,9 @@ body: |
; GFX8-NEXT: [[LSHR3:%[0-9]+]]:_(s16) = G_LSHR [[SHL1]], [[TRUNC3]](s16)
; GFX8-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC1]](s16), [[LSHR3]]
; GFX8-NEXT: [[SELECT1:%[0-9]+]]:_(s16) = G_SELECT [[ICMP1]](s1), [[C1]], [[SHL1]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL2]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -465,11 +464,11 @@ body: |
; GFX6-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL4]](s32), [[LSHR7]]
; GFX6-NEXT: [[SELECT2:%[0-9]+]]:_(s32) = G_SELECT [[ICMP2]](s1), [[C2]], [[SHL5]]
; GFX6-NEXT: [[LSHR8:%[0-9]+]]:_(s32) = G_LSHR [[SELECT2]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR6]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR6]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C4]], [[C1]](s32)
+ ; GFX6-NEXT: [[C4:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C4]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR8]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C4]], [[SHL7]]
@@ -508,18 +507,17 @@ body: |
; GFX8-NEXT: [[LSHR5:%[0-9]+]]:_(s16) = G_LSHR [[SHL2]], [[TRUNC5]](s16)
; GFX8-NEXT: [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC2]](s16), [[LSHR5]]
; GFX8-NEXT: [[SELECT2:%[0-9]+]]:_(s16) = G_SELECT [[ICMP2]](s1), [[C1]], [[SHL2]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT2]](s16)
- ; GFX8-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT2]](s16)
+ ; GFX8-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL4]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -620,10 +618,10 @@ body: |
; GFX6-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SHL6]](s32), [[LSHR10]]
; GFX6-NEXT: [[SELECT3:%[0-9]+]]:_(s32) = G_SELECT [[ICMP3]](s1), [[C3]], [[SHL7]]
; GFX6-NEXT: [[LSHR11:%[0-9]+]]:_(s32) = G_LSHR [[SELECT3]], [[C2]](s32)
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR11]], [[C2]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR11]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR9]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -670,15 +668,14 @@ body: |
; GFX8-NEXT: [[LSHR7:%[0-9]+]]:_(s16) = G_LSHR [[SHL3]], [[TRUNC7]](s16)
; GFX8-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[TRUNC3]](s16), [[LSHR7]]
; GFX8-NEXT: [[SELECT3:%[0-9]+]]:_(s16) = G_SELECT [[ICMP3]](s1), [[C1]], [[SHL3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT1]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT1]](s16)
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL4]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[SELECT3]](s16)
- ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C2]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[SELECT3]](s16)
+ ; GFX8-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL5]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
index 2d769fb96b97a..c6becfb8b3136 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubo.mir
@@ -142,18 +142,18 @@ body: |
; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[LSHR]], [[LSHR1]]
; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[SUB1]], [[C1]]
; CHECK-NEXT: [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB1]](s32), [[AND3]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C2]]
+ ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND5]], [[C]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND4]], [[SHL]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND4:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND5:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND4]](s32), [[AND5]](s32)
+ ; CHECK-NEXT: [[AND6:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<2 x s32>) = G_BUILD_VECTOR [[AND6]](s32), [[AND7]](s32)
; CHECK-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
; CHECK-NEXT: $vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<2 x s32>)
%0:_(<2 x s16>) = COPY $vgpr0
@@ -200,25 +200,25 @@ body: |
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
; CHECK-NEXT: [[ANYEXT1:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP1]](s1)
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C2]]
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND8]], [[C]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND7]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[SUB2]], [[C2]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND9]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>), [[BITCAST6]](<2 x s16>)
; CHECK-NEXT: [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND7]](s32), [[AND8]](s32), [[AND9]](s32)
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C4]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C4]]
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C4]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<3 x s32>) = G_BUILD_VECTOR [[AND10]](s32), [[AND11]](s32), [[AND12]](s32)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
; CHECK-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[BUILD_VECTOR]](<3 x s32>)
%0:_(<6 x s16>) = COPY $vgpr0_vgpr1_vgpr2
@@ -272,16 +272,16 @@ body: |
; CHECK-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[LSHR1]], [[LSHR3]]
; CHECK-NEXT: [[AND7:%[0-9]+]]:_(s32) = G_AND [[SUB3]], [[C1]]
; CHECK-NEXT: [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[SUB3]](s32), [[AND7]]
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[AND2]](s32)
- ; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY [[AND3]](s32)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[COPY3]], [[C2]](s32)
- ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[COPY2]], [[SHL]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(i32) = G_AND [[SUB]], [[C2]]
+ ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(i32) = G_AND [[SUB1]], [[C2]]
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[AND9]], [[C]](i32)
+ ; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND8]], [[SHL]]
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY [[AND6]](s32)
- ; CHECK-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY [[AND7]](s32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY5]], [[C2]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[COPY4]], [[SHL1]]
+ ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(i32) = G_AND [[SUB2]], [[C2]]
+ ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(i32) = G_AND [[SUB3]], [[C2]]
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[AND11]], [[C]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[AND10]], [[SHL1]]
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP]](s1)
@@ -289,11 +289,11 @@ body: |
; CHECK-NEXT: [[ANYEXT2:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP2]](s1)
; CHECK-NEXT: [[ANYEXT3:%[0-9]+]]:_(s32) = G_ANYEXT [[ICMP3]](s1)
; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
- ; CHECK-NEXT: [[AND8:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
- ; CHECK-NEXT: [[AND9:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
- ; CHECK-NEXT: [[AND10:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
- ; CHECK-NEXT: [[AND11:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
- ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND8]](s32), [[AND9]](s32), [[AND10]](s32), [[AND11]](s32)
+ ; CHECK-NEXT: [[AND12:%[0-9]+]]:_(s32) = G_AND [[ANYEXT]], [[C3]]
+ ; CHECK-NEXT: [[AND13:%[0-9]+]]:_(s32) = G_AND [[ANYEXT1]], [[C3]]
+ ; CHECK-NEXT: [[AND14:%[0-9]+]]:_(s32) = G_AND [[ANYEXT2]], [[C3]]
+ ; CHECK-NEXT: [[AND15:%[0-9]+]]:_(s32) = G_AND [[ANYEXT3]], [[C3]]
+ ; CHECK-NEXT: [[BUILD_VECTOR:%[0-9]+]]:_(<4 x s32>) = G_BUILD_VECTOR [[AND12]](s32), [[AND13]](s32), [[AND14]](s32), [[AND15]](s32)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[CONCAT_VECTORS]](<4 x s16>)
; CHECK-NEXT: $vgpr2_vgpr3_vgpr4_vgpr5 = COPY [[BUILD_VECTOR]](<4 x s32>)
%0:_(<4 x s16>) = COPY $vgpr0_vgpr1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
index 32e7fe682c681..a58918a7bce11 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-usubsat.mir
@@ -298,7 +298,7 @@ body: |
; GFX6-NEXT: [[UMIN1:%[0-9]+]]:_(s32) = G_UMIN [[SHL2]], [[SHL3]]
; GFX6-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[SHL2]], [[UMIN1]]
; GFX6-NEXT: [[LSHR3:%[0-9]+]]:_(s32) = G_LSHR [[SUB1]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR3]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR3]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR2]], [[SHL4]]
; GFX6-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX6-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -319,10 +319,9 @@ body: |
; GFX8-NEXT: [[TRUNC3:%[0-9]+]]:_(s16) = G_TRUNC [[LSHR1]](i32)
; GFX8-NEXT: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC2]]
; GFX8-NEXT: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC3]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST2]](<2 x s16>)
@@ -374,11 +373,11 @@ body: |
; GFX6-NEXT: [[UMIN2:%[0-9]+]]:_(s32) = G_UMIN [[SHL4]], [[SHL5]]
; GFX6-NEXT: [[SUB2:%[0-9]+]]:_(s32) = G_SUB [[SHL4]], [[UMIN2]]
; GFX6-NEXT: [[LSHR5:%[0-9]+]]:_(s32) = G_LSHR [[SUB2]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[LSHR4]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[LSHR4]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR3]], [[SHL6]]
; GFX6-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX6-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX6-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C2]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR5]], [[SHL7]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL7]]
@@ -407,18 +406,17 @@ body: |
; GFX8-NEXT: [[USUBSAT:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC]], [[TRUNC3]]
; GFX8-NEXT: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC4]]
; GFX8-NEXT: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC2]], [[TRUNC5]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT2]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT2]](s16)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C1]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[BITCAST3]](<2 x s16>), [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
; GFX8-NEXT: $vgpr0_vgpr1_vgpr2 = COPY [[CONCAT_VECTORS]](<6 x s16>)
@@ -503,10 +501,10 @@ body: |
; GFX6-NEXT: [[UMIN3:%[0-9]+]]:_(s32) = G_UMIN [[SHL6]], [[SHL7]]
; GFX6-NEXT: [[SUB3:%[0-9]+]]:_(s32) = G_SUB [[SHL6]], [[UMIN3]]
; GFX6-NEXT: [[LSHR7:%[0-9]+]]:_(s32) = G_LSHR [[SUB3]], [[C1]](s32)
- ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR5]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR5]], [[C]](i32)
; GFX6-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[LSHR4]], [[SHL8]]
; GFX6-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[LSHR7]], [[C1]](s32)
+ ; GFX6-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[LSHR7]], [[C]](i32)
; GFX6-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[LSHR6]], [[SHL9]]
; GFX6-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX6-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
@@ -540,15 +538,14 @@ body: |
; GFX8-NEXT: [[USUBSAT1:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC1]], [[TRUNC5]]
; GFX8-NEXT: [[USUBSAT2:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC2]], [[TRUNC6]]
; GFX8-NEXT: [[USUBSAT3:%[0-9]+]]:_(s16) = G_USUBSAT [[TRUNC3]], [[TRUNC7]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[USUBSAT3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[USUBSAT3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST4]](<2 x s16>), [[BITCAST5]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir
index 96549f71e2352..45c61ad7a2c35 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-vector-args-gfx8-plus.mir
@@ -92,15 +92,14 @@ body: |
; GFX8-NEXT: [[ADD:%[0-9]+]]:_(s16) = G_ADD [[TRUNC]], [[TRUNC3]]
; GFX8-NEXT: [[ADD1:%[0-9]+]]:_(s16) = G_ADD [[TRUNC1]], [[TRUNC4]]
; GFX8-NEXT: [[ADD2:%[0-9]+]]:_(s16) = G_ADD [[TRUNC2]], [[TRUNC5]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[ADD]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[ADD1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[ADD]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[ADD1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ADD2]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[ADD2]](s16)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST4]](<2 x s16>)
@@ -168,15 +167,14 @@ body: |
; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s16) = G_SHL [[TRUNC]], [[TRUNC3]](s16)
; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s16) = G_SHL [[TRUNC1]], [[TRUNC4]](s16)
; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s16) = G_SHL [[TRUNC2]], [[TRUNC5]](s16)
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[SHL]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[SHL1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[SHL]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[SHL1]](s16)
+ ; GFX8-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
; GFX8-NEXT: [[BITCAST4:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[SHL2]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[SHL2]](s16)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; GFX8-NEXT: [[BITCAST5:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST4]](<2 x s16>)
@@ -265,15 +263,14 @@ body: |
; GFX8-NEXT: [[FMA1:%[0-9]+]]:_(s16) = G_FMA [[TRUNC1]], [[TRUNC5]], [[TRUNC9]]
; GFX8-NEXT: [[FMA2:%[0-9]+]]:_(s16) = G_FMA [[TRUNC2]], [[TRUNC6]], [[TRUNC10]]
; GFX8-NEXT: [[FMA3:%[0-9]+]]:_(s16) = G_FMA [[TRUNC3]], [[TRUNC7]], [[TRUNC11]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMA]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMA1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMA]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMA1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMA2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMA3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMA2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMA3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
@@ -361,20 +358,19 @@ body: |
; GFX8-NEXT: [[FCANONICALIZE8:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC4]]
; GFX8-NEXT: [[FCANONICALIZE9:%[0-9]+]]:_(s16) = G_FCANONICALIZE [[TRUNC9]]
; GFX8-NEXT: [[FMAXNUM_IEEE4:%[0-9]+]]:_(s16) = G_FMAXNUM_IEEE [[FCANONICALIZE8]], [[FCANONICALIZE9]]
- ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
- ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
- ; GFX8-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE]](s16)
+ ; GFX8-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE1]](s16)
+ ; GFX8-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C]](i32)
; GFX8-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL]]
; GFX8-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
- ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE3]](s16)
- ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE2]](s16)
+ ; GFX8-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE3]](s16)
+ ; GFX8-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C]](i32)
; GFX8-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL1]]
; GFX8-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
- ; GFX8-NEXT: [[ZEXT4:%[0-9]+]]:_(s32) = G_ZEXT [[FMAXNUM_IEEE4]](s16)
- ; GFX8-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C1]](s32)
+ ; GFX8-NEXT: [[ZEXT4:%[0-9]+]]:_(i32) = G_ZEXT [[FMAXNUM_IEEE4]](s16)
+ ; GFX8-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; GFX8-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C1]], [[C]](i32)
; GFX8-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[ZEXT4]], [[SHL2]]
; GFX8-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
; GFX8-NEXT: $vgpr0 = COPY [[BITCAST6]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
index 8fcebe78c4cfe..b21e78e487e08 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-xor.mir
@@ -456,9 +456,9 @@ body: |
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[UV3]](<2 x s16>), [[UV4]](<2 x s16>)
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV6:%[0-9]+]]:_(<2 x s16>), [[UV7:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[XOR]](<4 x s16>)
- ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<6 x s16>) = G_CONCAT_VECTORS [[UV6]](<2 x s16>), [[UV7]](<2 x s16>), [[BITCAST]](<2 x s16>)
@@ -498,60 +498,59 @@ body: |
bb.0:
; CHECK-LABEL: name: test_xor_v5s16
- ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK: [[C:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL]]
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](i32)
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL1]]
; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR1]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST]](<2 x s16>), [[BITCAST1]](<2 x s16>)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL2]]
; CHECK-NEXT: [[BITCAST2:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR2]](i32)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL3]]
; CHECK-NEXT: [[BITCAST3:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR3]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS1:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST2]](<2 x s16>), [[BITCAST3]](<2 x s16>)
; CHECK-NEXT: [[XOR:%[0-9]+]]:_(<4 x s16>) = G_XOR [[CONCAT_VECTORS]], [[CONCAT_VECTORS1]]
; CHECK-NEXT: [[UV:%[0-9]+]]:_(<2 x s16>), [[UV1:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[XOR]](<4 x s16>)
; CHECK-NEXT: [[BITCAST4:%[0-9]+]]:_(i32) = G_BITCAST [[UV]](<2 x s16>)
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C2]](i32)
+ ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST4]], [[C1]](i32)
; CHECK-NEXT: [[BITCAST5:%[0-9]+]]:_(i32) = G_BITCAST [[UV1]](<2 x s16>)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL4]]
; CHECK-NEXT: [[BITCAST6:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR4]](i32)
- ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL5:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL5]]
; CHECK-NEXT: [[BITCAST7:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR5]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS2:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST6]](<2 x s16>), [[BITCAST7]](<2 x s16>)
- ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL6:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL6]]
; CHECK-NEXT: [[BITCAST8:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR6]](i32)
- ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL7:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR7:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL7]]
; CHECK-NEXT: [[BITCAST9:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR7]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS3:%[0-9]+]]:_(<4 x s16>) = G_CONCAT_VECTORS [[BITCAST8]](<2 x s16>), [[BITCAST9]](<2 x s16>)
; CHECK-NEXT: [[XOR1:%[0-9]+]]:_(<4 x s16>) = G_XOR [[CONCAT_VECTORS2]], [[CONCAT_VECTORS3]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(<2 x s16>), [[UV3:%[0-9]+]]:_(<2 x s16>) = G_UNMERGE_VALUES [[XOR1]](<4 x s16>)
; CHECK-NEXT: [[BITCAST10:%[0-9]+]]:_(i32) = G_BITCAST [[UV2]](<2 x s16>)
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C2]](i32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST4]], [[C3]]
- ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C1]](s32)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[BITCAST10]], [[C1]](i32)
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[BITCAST4]], [[C2]]
+ ; CHECK-NEXT: [[SHL8:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C1]](i32)
; CHECK-NEXT: [[OR8:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL8]]
; CHECK-NEXT: [[BITCAST11:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR8]](i32)
- ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[BITCAST5]], [[C3]]
- ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[BITCAST10]], [[C3]]
- ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C1]](s32)
+ ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(i32) = G_AND [[BITCAST5]], [[C2]]
+ ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i32) = G_AND [[BITCAST10]], [[C2]]
+ ; CHECK-NEXT: [[SHL9:%[0-9]+]]:_(i32) = G_SHL [[AND2]], [[C1]](i32)
; CHECK-NEXT: [[OR9:%[0-9]+]]:_(i32) = G_OR [[AND1]], [[SHL9]]
; CHECK-NEXT: [[BITCAST12:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR9]](i32)
- ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL10:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR10:%[0-9]+]]:_(i32) = G_OR [[LSHR1]], [[SHL10]]
; CHECK-NEXT: [[BITCAST13:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR10]](i32)
- ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(s32) = G_SHL [[C]], [[C1]](s32)
+ ; CHECK-NEXT: [[SHL11:%[0-9]+]]:_(i32) = G_SHL [[C]], [[C1]](i32)
; CHECK-NEXT: [[OR11:%[0-9]+]]:_(i32) = G_OR [[C]], [[SHL11]]
; CHECK-NEXT: [[BITCAST14:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR11]](i32)
; CHECK-NEXT: [[CONCAT_VECTORS4:%[0-9]+]]:_(<8 x s16>) = G_CONCAT_VECTORS [[BITCAST11]](<2 x s16>), [[BITCAST12]](<2 x s16>), [[BITCAST13]](<2 x s16>), [[BITCAST14]](<2 x s16>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
index 16a7255393262..caaa11ed1a96c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-zext.mir
@@ -667,17 +667,18 @@ body: |
; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i16) = G_SHL [[C5]], [[C4]](i16)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i16) = G_OR [[C5]], [[SHL2]]
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(i16) = COPY [[OR2]](i16)
- ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(s32) = G_ZEXT [[OR]](i16)
- ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](i16)
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[ZEXT1]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT:%[0-9]+]]:_(i32) = G_ZEXT [[OR]](i16)
+ ; CHECK-NEXT: [[ZEXT1:%[0-9]+]]:_(i32) = G_ZEXT [[OR1]](i16)
+ ; CHECK-NEXT: [[C6:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[ZEXT1]], [[C6]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[ZEXT]], [[SHL3]]
- ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[OR2]](i16)
- ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(s32) = G_ZEXT [[COPY1]](i16)
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[ZEXT3]], [[C1]](s32)
+ ; CHECK-NEXT: [[ZEXT2:%[0-9]+]]:_(i32) = G_ZEXT [[OR2]](i16)
+ ; CHECK-NEXT: [[ZEXT3:%[0-9]+]]:_(i32) = G_ZEXT [[COPY1]](i16)
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[ZEXT3]], [[C6]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[ZEXT2]], [[SHL4]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR3]](i32), [[OR4]](i32)
- ; CHECK-NEXT: [[C6:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s704) = G_MERGE_VALUES [[MV]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64), [[C6]](s64)
+ ; CHECK-NEXT: [[C7:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s704) = G_MERGE_VALUES [[MV]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64), [[C7]](s64)
; CHECK-NEXT: [[TRUNC4:%[0-9]+]]:_(s88) = G_TRUNC [[MV1]](s704)
; CHECK-NEXT: S_ENDPGM 0, implicit [[TRUNC4]](s88)
%0:_(s32) = COPY $vgpr0
@@ -711,37 +712,37 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16
; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32)
- ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535
- ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C1]]
- ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[LSHR]], [[C]](s32)
+ ; CHECK-NEXT: [[C1:%[0-9]+]]:_(i32) = G_CONSTANT i32 65535
+ ; CHECK-NEXT: [[AND:%[0-9]+]]:_(i32) = G_AND [[COPY]], [[C1]]
+ ; CHECK-NEXT: [[C2:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
+ ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(i32) = G_SHL [[LSHR]], [[C2]](i32)
; CHECK-NEXT: [[OR:%[0-9]+]]:_(i32) = G_OR [[AND]], [[SHL]]
- ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
- ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
- ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL1]]
+ ; CHECK-NEXT: [[C3:%[0-9]+]]:_(i32) = G_CONSTANT i32 0
+ ; CHECK-NEXT: [[SHL1:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C2]](i32)
+ ; CHECK-NEXT: [[OR1:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL1]]
; CHECK-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR]](i32), [[OR1]](i32)
- ; CHECK-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
- ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
- ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C4]](s64)
- ; CHECK-NEXT: [[C5:%[0-9]+]]:_(i32) = G_CONSTANT i32 16
- ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[UV]], [[C5]](i32)
- ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(s32) = G_SHL [[LSHR1]], [[C]](s32)
+ ; CHECK-NEXT: [[C4:%[0-9]+]]:_(s64) = G_CONSTANT i64 3
+ ; CHECK-NEXT: [[C5:%[0-9]+]]:_(s64) = G_CONSTANT i64 0
+ ; CHECK-NEXT: [[UV:%[0-9]+]]:_(i32), [[UV1:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[C5]](s64)
+ ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(i32) = G_LSHR [[UV]], [[C2]](i32)
+ ; CHECK-NEXT: [[SHL2:%[0-9]+]]:_(i32) = G_SHL [[LSHR1]], [[C2]](i32)
; CHECK-NEXT: [[OR2:%[0-9]+]]:_(i32) = G_OR [[UV]], [[SHL2]]
- ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(s32) = G_SHL [[C2]], [[C]](s32)
+ ; CHECK-NEXT: [[SHL3:%[0-9]+]]:_(i32) = G_SHL [[C3]], [[C2]](i32)
; CHECK-NEXT: [[OR3:%[0-9]+]]:_(i32) = G_OR [[UV1]], [[SHL3]]
; CHECK-NEXT: [[MV1:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[OR2]](i32), [[OR3]](i32)
; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY [[MV]](s64)
- ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[C3]](s64)
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY [[C4]](s64)
; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s64) = G_AND [[COPY1]], [[COPY2]]
; CHECK-NEXT: [[DEF:%[0-9]+]]:_(i64) = G_IMPLICIT_DEF
; CHECK-NEXT: [[AND2:%[0-9]+]]:_(i64) = G_AND [[DEF]], [[MV1]]
; CHECK-NEXT: [[UV2:%[0-9]+]]:_(i32), [[UV3:%[0-9]+]]:_(i32) = G_UNMERGE_VALUES [[AND2]](i64)
- ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[UV2]], [[C5]](i32)
- ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(s32) = G_AND [[UV2]], [[C1]]
- ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(s32) = G_SHL [[LSHR2]], [[C]](s32)
+ ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(i32) = G_LSHR [[UV2]], [[C2]](i32)
+ ; CHECK-NEXT: [[AND3:%[0-9]+]]:_(i32) = G_AND [[UV2]], [[C1]]
+ ; CHECK-NEXT: [[SHL4:%[0-9]+]]:_(i32) = G_SHL [[LSHR2]], [[C2]](i32)
; CHECK-NEXT: [[OR4:%[0-9]+]]:_(i32) = G_OR [[AND3]], [[SHL4]]
; CHECK-NEXT: [[OR5:%[0-9]+]]:_(i32) = G_OR [[UV3]], [[SHL3]]
; CHECK-NEXT: [[MV2:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR4]](i32), [[OR5]](i32)
- ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[C2]], [[SHL3]]
+ ; CHECK-NEXT: [[OR6:%[0-9]+]]:_(i32) = G_OR [[C3]], [[SHL3]]
; CHECK-NEXT: [[COPY3:%[0-9]+]]:_(i32) = COPY [[OR6]](i32)
; CHECK-NEXT: [[MV3:%[0-9]+]]:_(i64) = G_MERGE_VALUES [[OR6]](i32), [[COPY3]](i32)
; CHECK-NEXT: [[COPY4:%[0-9]+]]:_(i32) = COPY [[OR6]](i32)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
index 09d8cfcca7460..ecc0f74b2efef 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.abs.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX8
+; RUN: llc -global-isel -mtriple=amdgpu6.00 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX6
+; RUN: llc -global-isel -mtriple=amdgpu8.03 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX8
; RUN: llc -global-isel -mtriple=amdgpu10.10 -o - < %s | FileCheck %s --check-prefixes=GFX,GFX10
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=-real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-FAKE16
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.50 -mattr=+real-true16 -o - < %s | FileCheck %s --check-prefixes=GFX1250,GFX1250-REAL16
@@ -479,21 +479,22 @@ define <3 x i8> @abs_vgpr_v3i8(<3 x i8> %arg) {
define amdgpu_cs <2 x i16> @abs_sgpr_v2i16(<2 x i16> inreg %arg) {
; GFX6-LABEL: abs_sgpr_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s1, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: s_abs_i32 s1, s1
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_sext_i32_i16 s1, s0
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100010
; GFX6-NEXT: s_abs_i32 s0, s0
-; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_abs_i32 s1, s1
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_or_b32 s0, s1, s0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: abs_sgpr_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_ashr_i32 s1, s0, 16
-; GFX8-NEXT: s_abs_i32 s1, s1
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_abs_i32 s1, s1
; GFX8-NEXT: s_abs_i32 s0, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -526,25 +527,25 @@ define <2 x i16> @abs_vgpr_v2i16(<2 x i16> %arg) {
; GFX6-LABEL: abs_vgpr_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_i32 v1, v0, 0, 16
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 0, v1
-; GFX6-NEXT: v_max_i32_e32 v1, v2, v1
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX6-NEXT: v_max_i32_e32 v1, v1, v2
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_max_i32_e32 v0, v2, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_max_i32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: abs_vgpr_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e32 v2, 0, v0
-; GFX8-NEXT: v_max_i16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_sub_u16_e32 v1, 0, v0
+; GFX8-NEXT: v_sub_u16_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v1, v0, v1
+; GFX8-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: abs_vgpr_v2i16:
@@ -569,23 +570,24 @@ define <2 x i16> @abs_vgpr_v2i16(<2 x i16> %arg) {
define amdgpu_cs <3 x i16> @abs_sgpr_v3i16(<3 x i16> inreg %arg) {
; GFX6-LABEL: abs_sgpr_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s2, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
+; GFX6-NEXT: s_sext_i32_i16 s2, s0
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x100010
+; GFX6-NEXT: s_abs_i32 s0, s0
; GFX6-NEXT: s_abs_i32 s2, s2
; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_lshl_b32 s2, s2, 16
-; GFX6-NEXT: s_abs_i32 s0, s0
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
; GFX6-NEXT: s_abs_i32 s1, s1
-; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_or_b32 s0, s2, s0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: abs_sgpr_v3i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_ashr_i32 s2, s0, 16
-; GFX8-NEXT: s_abs_i32 s2, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_abs_i32 s2, s2
; GFX8-NEXT: s_abs_i32 s0, s0
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
; GFX8-NEXT: s_lshl_b32 s2, s2, 16
; GFX8-NEXT: s_abs_i32 s1, s1
; GFX8-NEXT: s_or_b32 s0, s0, s2
@@ -623,30 +625,30 @@ define <3 x i16> @abs_vgpr_v3i16(<3 x i16> %arg) {
; GFX6-LABEL: abs_vgpr_v3i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v0
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_i32 v2, v0, 0, 16
; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 0, v2
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_max_i32_e32 v2, v3, v2
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 0, v1
-; GFX6-NEXT: v_max_i32_e32 v1, v3, v1
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 16
+; GFX6-NEXT: v_max_i32_e32 v2, v2, v3
; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_max_i32_e32 v0, v3, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_max_i32_e32 v0, v0, v3
+; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 0, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v1, v1, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: abs_vgpr_v3i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, 0
-; GFX8-NEXT: v_sub_u16_e32 v3, 0, v1
-; GFX8-NEXT: v_sub_u16_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_max_i16_e32 v1, v1, v3
-; GFX8-NEXT: v_sub_u16_e32 v3, 0, v0
-; GFX8-NEXT: v_max_i16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v0, v0, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_sub_u16_e32 v2, 0, v0
+; GFX8-NEXT: v_sub_u16_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e32 v4, 0, v1
+; GFX8-NEXT: v_max_i16_e32 v2, v0, v2
+; GFX8-NEXT: v_max_i16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-NEXT: v_max_i16_e32 v1, v1, v4
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: abs_vgpr_v3i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
index cac0a7017dc32..0c1195ff3f640 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.load.1d.d16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d < %s | FileCheck -check-prefix=GFX8-UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d < %s | FileCheck -check-prefix=GFX8-UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d < %s | FileCheck -check-prefix=GFX8-PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10 %s
@@ -361,16 +361,17 @@ define amdgpu_ps half @load_1d_f16_w(<8 x i32> inreg %rsrc, i32 %s) {
define amdgpu_ps <2 x half> @load_1d_v2f16_xy(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-LABEL: load_1d_v2f16_xy:
; GFX8-UNPACKED: ; %bb.0:
-; GFX8-UNPACKED-NEXT: s_mov_b32 s11, s9
-; GFX8-UNPACKED-NEXT: s_mov_b32 s10, s8
-; GFX8-UNPACKED-NEXT: s_mov_b32 s9, s7
-; GFX8-UNPACKED-NEXT: s_mov_b32 s8, s6
-; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s5
-; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s4
-; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s3
-; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s2
-; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[4:11] dmask:0x3 unorm d16
+; GFX8-UNPACKED-NEXT: s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT: s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT: s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT: s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[0:7] dmask:0x3 unorm d16
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: ; return to shader part epilog
@@ -451,16 +452,17 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_xy(<8 x i32> inreg %rsrc, i32 %s) {
define amdgpu_ps <2 x half> @load_1d_v2f16_xz(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-LABEL: load_1d_v2f16_xz:
; GFX8-UNPACKED: ; %bb.0:
-; GFX8-UNPACKED-NEXT: s_mov_b32 s11, s9
-; GFX8-UNPACKED-NEXT: s_mov_b32 s10, s8
-; GFX8-UNPACKED-NEXT: s_mov_b32 s9, s7
-; GFX8-UNPACKED-NEXT: s_mov_b32 s8, s6
-; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s5
-; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s4
-; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s3
-; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s2
-; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[4:11] dmask:0x5 unorm d16
+; GFX8-UNPACKED-NEXT: s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT: s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT: s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT: s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[0:7] dmask:0x5 unorm d16
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: ; return to shader part epilog
@@ -541,16 +543,17 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_xz(<8 x i32> inreg %rsrc, i32 %s) {
define amdgpu_ps <2 x half> @load_1d_v2f16_xw(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-LABEL: load_1d_v2f16_xw:
; GFX8-UNPACKED: ; %bb.0:
-; GFX8-UNPACKED-NEXT: s_mov_b32 s11, s9
-; GFX8-UNPACKED-NEXT: s_mov_b32 s10, s8
-; GFX8-UNPACKED-NEXT: s_mov_b32 s9, s7
-; GFX8-UNPACKED-NEXT: s_mov_b32 s8, s6
-; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s5
-; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s4
-; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s3
-; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s2
-; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[4:11] dmask:0x9 unorm d16
+; GFX8-UNPACKED-NEXT: s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT: s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT: s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT: s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[0:7] dmask:0x9 unorm d16
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: ; return to shader part epilog
@@ -631,16 +634,17 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_xw(<8 x i32> inreg %rsrc, i32 %s) {
define amdgpu_ps <2 x half> @load_1d_v2f16_yz(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-LABEL: load_1d_v2f16_yz:
; GFX8-UNPACKED: ; %bb.0:
-; GFX8-UNPACKED-NEXT: s_mov_b32 s11, s9
-; GFX8-UNPACKED-NEXT: s_mov_b32 s10, s8
-; GFX8-UNPACKED-NEXT: s_mov_b32 s9, s7
-; GFX8-UNPACKED-NEXT: s_mov_b32 s8, s6
-; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s5
-; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s4
-; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s3
-; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s2
-; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[4:11] dmask:0x6 unorm d16
+; GFX8-UNPACKED-NEXT: s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT: s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT: s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT: s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT: image_load v[0:1], v0, s[0:7] dmask:0x6 unorm d16
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: ; return to shader part epilog
@@ -721,19 +725,20 @@ define amdgpu_ps <2 x half> @load_1d_v2f16_yz(<8 x i32> inreg %rsrc, i32 %s) {
define amdgpu_ps <3 x half> @load_1d_v3f16_xyz(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-LABEL: load_1d_v3f16_xyz:
; GFX8-UNPACKED: ; %bb.0:
-; GFX8-UNPACKED-NEXT: s_mov_b32 s11, s9
-; GFX8-UNPACKED-NEXT: s_mov_b32 s10, s8
-; GFX8-UNPACKED-NEXT: s_mov_b32 s9, s7
-; GFX8-UNPACKED-NEXT: s_mov_b32 s8, s6
-; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s5
-; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s4
-; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s3
-; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s2
-; GFX8-UNPACKED-NEXT: image_load v[0:2], v0, s[4:11] dmask:0x7 unorm d16
-; GFX8-UNPACKED-NEXT: s_mov_b32 s0, 0x1000504
+; GFX8-UNPACKED-NEXT: s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT: s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT: s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT: s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT: image_load v[0:2], v0, s[0:7] dmask:0x7 unorm d16
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX8-UNPACKED-NEXT: v_perm_b32 v0, v0, v1, s0
-; GFX8-UNPACKED-NEXT: v_mov_b32_e32 v1, v2
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: ; return to shader part epilog
;
; GFX8-PACKED-LABEL: load_1d_v3f16_xyz:
@@ -813,19 +818,22 @@ define amdgpu_ps <3 x half> @load_1d_v3f16_xyz(<8 x i32> inreg %rsrc, i32 %s) {
define amdgpu_ps <4 x half> @load_1d_v4f16_xyzw(<8 x i32> inreg %rsrc, i32 %s) {
; GFX8-UNPACKED-LABEL: load_1d_v4f16_xyzw:
; GFX8-UNPACKED: ; %bb.0:
-; GFX8-UNPACKED-NEXT: s_mov_b32 s11, s9
-; GFX8-UNPACKED-NEXT: s_mov_b32 s10, s8
-; GFX8-UNPACKED-NEXT: s_mov_b32 s9, s7
-; GFX8-UNPACKED-NEXT: s_mov_b32 s8, s6
-; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s5
-; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s4
-; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s3
-; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s2
-; GFX8-UNPACKED-NEXT: image_load v[0:3], v0, s[4:11] dmask:0xf unorm d16
-; GFX8-UNPACKED-NEXT: s_mov_b32 s0, 0x1000504
+; GFX8-UNPACKED-NEXT: s_mov_b32 s0, s2
+; GFX8-UNPACKED-NEXT: s_mov_b32 s1, s3
+; GFX8-UNPACKED-NEXT: s_mov_b32 s2, s4
+; GFX8-UNPACKED-NEXT: s_mov_b32 s3, s5
+; GFX8-UNPACKED-NEXT: s_mov_b32 s4, s6
+; GFX8-UNPACKED-NEXT: s_mov_b32 s5, s7
+; GFX8-UNPACKED-NEXT: s_mov_b32 s6, s8
+; GFX8-UNPACKED-NEXT: s_mov_b32 s7, s9
+; GFX8-UNPACKED-NEXT: image_load v[0:3], v0, s[0:7] dmask:0xf unorm d16
; GFX8-UNPACKED-NEXT: s_waitcnt vmcnt(0)
-; GFX8-UNPACKED-NEXT: v_perm_b32 v0, v0, v1, s0
-; GFX8-UNPACKED-NEXT: v_perm_b32 v1, v2, v3, s0
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-UNPACKED-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-UNPACKED-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-UNPACKED-NEXT: v_or_b32_sdwa v1, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-UNPACKED-NEXT: ; return to shader part epilog
;
; GFX8-PACKED-LABEL: load_1d_v4f16_xyzw:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
index 6626365daab67..d186850117193 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.store.2d.d16.ll
@@ -87,15 +87,20 @@ define amdgpu_ps void @image_store_v3f16(<8 x i32> inreg %rsrc, i32 %s, i32 %t,
;
; GFX81-LABEL: image_store_v3f16:
; GFX81: ; %bb.0:
-; GFX81-NEXT: s_mov_b32 s11, s9
-; GFX81-NEXT: s_mov_b32 s10, s8
-; GFX81-NEXT: s_mov_b32 s9, s7
-; GFX81-NEXT: s_mov_b32 s8, s6
-; GFX81-NEXT: s_mov_b32 s7, s5
-; GFX81-NEXT: s_mov_b32 s6, s4
-; GFX81-NEXT: s_mov_b32 s5, s3
-; GFX81-NEXT: s_mov_b32 s4, s2
-; GFX81-NEXT: image_store v[2:4], v[0:1], s[4:11] dmask:0x7 unorm d16
+; GFX81-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX81-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX81-NEXT: s_mov_b32 s0, s2
+; GFX81-NEXT: s_mov_b32 s1, s3
+; GFX81-NEXT: s_mov_b32 s2, s4
+; GFX81-NEXT: s_mov_b32 s3, s5
+; GFX81-NEXT: s_mov_b32 s4, s6
+; GFX81-NEXT: s_mov_b32 s5, s7
+; GFX81-NEXT: s_mov_b32 s6, s8
+; GFX81-NEXT: s_mov_b32 s7, s9
+; GFX81-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX81-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX81-NEXT: v_mov_b32_e32 v4, 0
+; GFX81-NEXT: image_store v[2:4], v[0:1], s[0:7] dmask:0x7 unorm d16
; GFX81-NEXT: s_endpgm
call void @llvm.amdgcn.image.store.2d.v3f16.i32(<3 x half> %in, i32 7, i32 %s, i32 %t, <8 x i32> %rsrc, i32 0, i32 0)
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index ea69f58ead7d4..1d0ae809d3707 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1030 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1013 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1030 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=GCN,GFX10,GFX1013 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-FAKE16 %s
; RUN: not llc -global-isel -mtriple=amdgpu10.12 < %s -filetype=null 2>&1 | FileCheck -check-prefix=ERR %s
@@ -51,8 +51,14 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_flat(i32 %node_ptr, float
define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> inreg %tdescr) {
; GFX10-LABEL: image_bvh_intersect_ray_a16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_perm_b32 v6, v7, v6, 0x5040100
+; GFX10-NEXT: v_lshrrev_b32_e32 v9, 16, v5
+; GFX10-NEXT: v_and_b32_e32 v10, 0xffff, v7
+; GFX10-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX10-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; GFX10-NEXT: v_alignbit_b32 v7, v8, v7, 16
+; GFX10-NEXT: v_and_or_b32 v5, 0xffff, v5, v9
+; GFX10-NEXT: v_and_or_b32 v6, 0xffff, v6, v10
; GFX10-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[0:3] a16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -120,8 +126,14 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_flat(<2 x i32> %node_ptr
define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> inreg %tdescr) {
; GFX10-LABEL: image_bvh64_intersect_ray_a16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: v_perm_b32 v7, v8, v7, 0x5040100
+; GFX10-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; GFX10-NEXT: v_and_b32_e32 v11, 0xffff, v8
+; GFX10-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX10-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX10-NEXT: v_lshlrev_b32_e32 v11, 16, v11
; GFX10-NEXT: v_alignbit_b32 v8, v9, v8, 16
+; GFX10-NEXT: v_and_or_b32 v6, 0xffff, v6, v10
+; GFX10-NEXT: v_and_or_b32 v7, 0xffff, v7, v11
; GFX10-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[0:3] a16
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: ; return to shader part epilog
@@ -266,26 +278,31 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_vgpr_descr(i32 %node_ptr,
define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr) {
; GFX1030-LABEL: image_bvh_intersect_ray_a16_vgpr_descr:
; GFX1030: ; %bb.0:
-; GFX1030-NEXT: v_mov_b32_e32 v18, v5
-; GFX1030-NEXT: v_mov_b32_e32 v17, v4
-; GFX1030-NEXT: v_mov_b32_e32 v16, v3
-; GFX1030-NEXT: v_mov_b32_e32 v15, v2
-; GFX1030-NEXT: v_mov_b32_e32 v14, v1
; GFX1030-NEXT: v_mov_b32_e32 v13, v0
-; GFX1030-NEXT: v_perm_b32 v19, v7, v6, 0x5040100
-; GFX1030-NEXT: v_alignbit_b32 v20, v8, v7, 16
-; GFX1030-NEXT: s_mov_b32 s4, exec_lo
-; GFX1030-NEXT: s_mov_b32 s5, s4
+; GFX1030-NEXT: v_mov_b32_e32 v14, v1
+; GFX1030-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX1030-NEXT: v_and_b32_e32 v1, 0xffff, v7
+; GFX1030-NEXT: v_mov_b32_e32 v15, v2
+; GFX1030-NEXT: v_and_b32_e32 v2, 0xffff, v8
+; GFX1030-NEXT: v_mov_b32_e32 v16, v3
+; GFX1030-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1030-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX1030-NEXT: v_mov_b32_e32 v17, v4
+; GFX1030-NEXT: v_alignbit_b32 v20, v2, v7, 16
+; GFX1030-NEXT: s_mov_b32 s1, exec_lo
+; GFX1030-NEXT: v_and_or_b32 v18, 0xffff, v5, v0
+; GFX1030-NEXT: v_and_or_b32 v19, 0xffff, v6, v1
; GFX1030-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX1030-NEXT: v_readfirstlane_b32 s0, v9
-; GFX1030-NEXT: v_readfirstlane_b32 s1, v10
-; GFX1030-NEXT: v_readfirstlane_b32 s2, v11
-; GFX1030-NEXT: v_readfirstlane_b32 s3, v12
-; GFX1030-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
-; GFX1030-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[11:12]
-; GFX1030-NEXT: image_bvh_intersect_ray v[0:3], v[13:20], s[0:3] a16
-; GFX1030-NEXT: s_andn2_wrexec_b32 s5, s5
-; GFX1030-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11_vgpr12
+; GFX1030-NEXT: v_readfirstlane_b32 s4, v9
+; GFX1030-NEXT: v_readfirstlane_b32 s5, v10
+; GFX1030-NEXT: v_readfirstlane_b32 s6, v11
+; GFX1030-NEXT: v_readfirstlane_b32 s7, v12
+; GFX1030-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX1030-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[11:12]
+; GFX1030-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1030-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1030-NEXT: image_bvh_intersect_ray v[0:3], v[13:20], s[4:7] a16
+; GFX1030-NEXT: ; implicit-def: $vgpr9
; GFX1030-NEXT: ; implicit-def: $vgpr13
; GFX1030-NEXT: ; implicit-def: $vgpr14
; GFX1030-NEXT: ; implicit-def: $vgpr15
@@ -294,34 +311,43 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
; GFX1030-NEXT: ; implicit-def: $vgpr18
; GFX1030-NEXT: ; implicit-def: $vgpr19
; GFX1030-NEXT: ; implicit-def: $vgpr20
+; GFX1030-NEXT: ; implicit-def: $vgpr11_vgpr12
+; GFX1030-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1030-NEXT: s_cbranch_execnz .LBB7_1
; GFX1030-NEXT: ; %bb.2:
-; GFX1030-NEXT: s_mov_b32 exec_lo, s4
+; GFX1030-NEXT: s_mov_b32 exec_lo, s1
; GFX1030-NEXT: s_waitcnt vmcnt(0)
; GFX1030-NEXT: ; return to shader part epilog
;
; GFX1013-LABEL: image_bvh_intersect_ray_a16_vgpr_descr:
; GFX1013: ; %bb.0:
-; GFX1013-NEXT: v_perm_b32 v6, v7, v6, 0x5040100
+; GFX1013-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX1013-NEXT: v_and_b32_e32 v14, 0xffff, v7
+; GFX1013-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX1013-NEXT: s_mov_b32 s1, exec_lo
+; GFX1013-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; GFX1013-NEXT: v_lshlrev_b32_e32 v14, 16, v14
; GFX1013-NEXT: v_alignbit_b32 v7, v8, v7, 16
-; GFX1013-NEXT: s_mov_b32 s4, exec_lo
-; GFX1013-NEXT: s_mov_b32 s5, s4
+; GFX1013-NEXT: v_and_or_b32 v5, 0xffff, v5, v13
+; GFX1013-NEXT: v_and_or_b32 v6, 0xffff, v6, v14
; GFX1013-NEXT: .LBB7_1: ; =>This Inner Loop Header: Depth=1
-; GFX1013-NEXT: v_readfirstlane_b32 s0, v9
-; GFX1013-NEXT: v_readfirstlane_b32 s1, v10
-; GFX1013-NEXT: v_readfirstlane_b32 s2, v11
-; GFX1013-NEXT: v_readfirstlane_b32 s3, v12
-; GFX1013-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1013-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[9:10]
-; GFX1013-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[11:12]
-; GFX1013-NEXT: image_bvh_intersect_ray v[13:16], v[0:7], s[0:3] a16
-; GFX1013-NEXT: s_andn2_wrexec_b32 s5, s5
-; GFX1013-NEXT: ; implicit-def: $vgpr9_vgpr10_vgpr11_vgpr12
+; GFX1013-NEXT: v_readfirstlane_b32 s4, v9
+; GFX1013-NEXT: v_readfirstlane_b32 s5, v10
+; GFX1013-NEXT: v_readfirstlane_b32 s6, v11
+; GFX1013-NEXT: v_readfirstlane_b32 s7, v12
+; GFX1013-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[9:10]
+; GFX1013-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[11:12]
+; GFX1013-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1013-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1013-NEXT: image_bvh_intersect_ray v[13:16], v[0:7], s[4:7] a16
+; GFX1013-NEXT: ; implicit-def: $vgpr9
; GFX1013-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7
+; GFX1013-NEXT: ; implicit-def: $vgpr11_vgpr12
+; GFX1013-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1013-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1013-NEXT: s_cbranch_execnz .LBB7_1
; GFX1013-NEXT: ; %bb.2:
-; GFX1013-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1013-NEXT: s_mov_b32 exec_lo, s4
+; GFX1013-NEXT: s_mov_b32 exec_lo, s1
; GFX1013-NEXT: s_waitcnt vmcnt(0)
; GFX1013-NEXT: v_mov_b32_e32 v0, v13
; GFX1013-NEXT: v_mov_b32_e32 v1, v14
@@ -519,27 +545,32 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_vgpr_descr(i64 %node_ptr
define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node_ptr, float %ray_extent, <3 x float> %ray_origin, <3 x half> %ray_dir, <3 x half> %ray_inv_dir, <4 x i32> %tdescr) {
; GFX1030-LABEL: image_bvh64_intersect_ray_a16_vgpr_descr:
; GFX1030: ; %bb.0:
-; GFX1030-NEXT: v_mov_b32_e32 v20, v6
-; GFX1030-NEXT: v_mov_b32_e32 v19, v5
-; GFX1030-NEXT: v_mov_b32_e32 v18, v4
-; GFX1030-NEXT: v_mov_b32_e32 v17, v3
-; GFX1030-NEXT: v_mov_b32_e32 v16, v2
-; GFX1030-NEXT: v_mov_b32_e32 v15, v1
; GFX1030-NEXT: v_mov_b32_e32 v14, v0
-; GFX1030-NEXT: v_perm_b32 v21, v8, v7, 0x5040100
-; GFX1030-NEXT: v_alignbit_b32 v22, v9, v8, 16
-; GFX1030-NEXT: s_mov_b32 s4, exec_lo
-; GFX1030-NEXT: s_mov_b32 s5, s4
+; GFX1030-NEXT: v_mov_b32_e32 v15, v1
+; GFX1030-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; GFX1030-NEXT: v_and_b32_e32 v1, 0xffff, v8
+; GFX1030-NEXT: v_mov_b32_e32 v16, v2
+; GFX1030-NEXT: v_and_b32_e32 v2, 0xffff, v9
+; GFX1030-NEXT: v_mov_b32_e32 v17, v3
+; GFX1030-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1030-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX1030-NEXT: v_mov_b32_e32 v18, v4
+; GFX1030-NEXT: v_mov_b32_e32 v19, v5
+; GFX1030-NEXT: v_alignbit_b32 v22, v2, v8, 16
+; GFX1030-NEXT: v_and_or_b32 v20, 0xffff, v6, v0
+; GFX1030-NEXT: v_and_or_b32 v21, 0xffff, v7, v1
+; GFX1030-NEXT: s_mov_b32 s1, exec_lo
; GFX1030-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1030-NEXT: v_readfirstlane_b32 s0, v10
-; GFX1030-NEXT: v_readfirstlane_b32 s1, v11
-; GFX1030-NEXT: v_readfirstlane_b32 s2, v12
-; GFX1030-NEXT: v_readfirstlane_b32 s3, v13
-; GFX1030-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
-; GFX1030-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
-; GFX1030-NEXT: image_bvh64_intersect_ray v[0:3], v[14:22], s[0:3] a16
-; GFX1030-NEXT: s_andn2_wrexec_b32 s5, s5
-; GFX1030-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
+; GFX1030-NEXT: v_readfirstlane_b32 s4, v10
+; GFX1030-NEXT: v_readfirstlane_b32 s5, v11
+; GFX1030-NEXT: v_readfirstlane_b32 s6, v12
+; GFX1030-NEXT: v_readfirstlane_b32 s7, v13
+; GFX1030-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[10:11]
+; GFX1030-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[12:13]
+; GFX1030-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1030-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1030-NEXT: image_bvh64_intersect_ray v[0:3], v[14:22], s[4:7] a16
+; GFX1030-NEXT: ; implicit-def: $vgpr10
; GFX1030-NEXT: ; implicit-def: $vgpr14
; GFX1030-NEXT: ; implicit-def: $vgpr15
; GFX1030-NEXT: ; implicit-def: $vgpr16
@@ -549,34 +580,43 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
; GFX1030-NEXT: ; implicit-def: $vgpr20
; GFX1030-NEXT: ; implicit-def: $vgpr21
; GFX1030-NEXT: ; implicit-def: $vgpr22
+; GFX1030-NEXT: ; implicit-def: $vgpr12_vgpr13
+; GFX1030-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1030-NEXT: s_cbranch_execnz .LBB9_1
; GFX1030-NEXT: ; %bb.2:
-; GFX1030-NEXT: s_mov_b32 exec_lo, s4
+; GFX1030-NEXT: s_mov_b32 exec_lo, s1
; GFX1030-NEXT: s_waitcnt vmcnt(0)
; GFX1030-NEXT: ; return to shader part epilog
;
; GFX1013-LABEL: image_bvh64_intersect_ray_a16_vgpr_descr:
; GFX1013: ; %bb.0:
-; GFX1013-NEXT: v_perm_b32 v7, v8, v7, 0x5040100
+; GFX1013-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX1013-NEXT: v_and_b32_e32 v15, 0xffff, v8
+; GFX1013-NEXT: v_and_b32_e32 v9, 0xffff, v9
+; GFX1013-NEXT: s_mov_b32 s1, exec_lo
+; GFX1013-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; GFX1013-NEXT: v_lshlrev_b32_e32 v15, 16, v15
; GFX1013-NEXT: v_alignbit_b32 v8, v9, v8, 16
-; GFX1013-NEXT: s_mov_b32 s4, exec_lo
-; GFX1013-NEXT: s_mov_b32 s5, s4
+; GFX1013-NEXT: v_and_or_b32 v6, 0xffff, v6, v14
+; GFX1013-NEXT: v_and_or_b32 v7, 0xffff, v7, v15
; GFX1013-NEXT: .LBB9_1: ; =>This Inner Loop Header: Depth=1
-; GFX1013-NEXT: v_readfirstlane_b32 s0, v10
-; GFX1013-NEXT: v_readfirstlane_b32 s1, v11
-; GFX1013-NEXT: v_readfirstlane_b32 s2, v12
-; GFX1013-NEXT: v_readfirstlane_b32 s3, v13
-; GFX1013-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX1013-NEXT: v_cmpx_eq_u64_e32 s[0:1], v[10:11]
-; GFX1013-NEXT: v_cmpx_eq_u64_e32 s[2:3], v[12:13]
-; GFX1013-NEXT: image_bvh64_intersect_ray v[14:17], v[0:8], s[0:3] a16
-; GFX1013-NEXT: s_andn2_wrexec_b32 s5, s5
-; GFX1013-NEXT: ; implicit-def: $vgpr10_vgpr11_vgpr12_vgpr13
+; GFX1013-NEXT: v_readfirstlane_b32 s4, v10
+; GFX1013-NEXT: v_readfirstlane_b32 s5, v11
+; GFX1013-NEXT: v_readfirstlane_b32 s6, v12
+; GFX1013-NEXT: v_readfirstlane_b32 s7, v13
+; GFX1013-NEXT: v_cmp_eq_u64_e32 vcc_lo, s[4:5], v[10:11]
+; GFX1013-NEXT: v_cmp_eq_u64_e64 s0, s[6:7], v[12:13]
+; GFX1013-NEXT: s_and_b32 s0, vcc_lo, s0
+; GFX1013-NEXT: s_and_saveexec_b32 s0, s0
+; GFX1013-NEXT: image_bvh64_intersect_ray v[14:17], v[0:8], s[4:7] a16
+; GFX1013-NEXT: ; implicit-def: $vgpr10
; GFX1013-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8
+; GFX1013-NEXT: ; implicit-def: $vgpr12_vgpr13
+; GFX1013-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
+; GFX1013-NEXT: s_xor_b32 exec_lo, exec_lo, s0
; GFX1013-NEXT: s_cbranch_execnz .LBB9_1
; GFX1013-NEXT: ; %bb.2:
-; GFX1013-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1013-NEXT: s_mov_b32 exec_lo, s4
+; GFX1013-NEXT: s_mov_b32 exec_lo, s1
; GFX1013-NEXT: s_waitcnt vmcnt(0)
; GFX1013-NEXT: v_mov_b32_e32 v0, v14
; GFX1013-NEXT: v_mov_b32_e32 v1, v15
@@ -782,20 +822,24 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1030-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX1030: ; %bb.0:
; GFX1030-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
-; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX1030-NEXT: v_mov_b32_e32 v7, 0x48004700
-; GFX1030-NEXT: v_mov_b32_e32 v6, 0x46004500
+; GFX1030-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1030-NEXT: v_mov_b32_e32 v5, 0x44004200
-; GFX1030-NEXT: v_mov_b32_e32 v4, 2.0
+; GFX1030-NEXT: v_mov_b32_e32 v6, 0x46004500
+; GFX1030-NEXT: v_mov_b32_e32 v7, 0x48004700
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_add_co_u32 v0, s0, s0, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1030-NEXT: v_add_co_u32 v2, s0, s2, v2
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
+; GFX1030-NEXT: v_mov_b32_e32 v0, s0
+; GFX1030-NEXT: v_mov_b32_e32 v1, s1
+; GFX1030-NEXT: v_mov_b32_e32 v2, s2
+; GFX1030-NEXT: v_mov_b32_e32 v3, s3
+; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1030-NEXT: flat_load_dword v0, v[0:1]
; GFX1030-NEXT: flat_load_dword v1, v[2:3]
; GFX1030-NEXT: v_mov_b32_e32 v2, 0
; GFX1030-NEXT: v_mov_b32_e32 v3, 1.0
+; GFX1030-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1030-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1030-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[4:7] a16
; GFX1030-NEXT: s_waitcnt vmcnt(0)
@@ -805,20 +849,24 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX1013: ; %bb.0:
; GFX1013-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX1013-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1013-NEXT: v_lshlrev_b32_e32 v6, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v7, 0x48004700
-; GFX1013-NEXT: v_mov_b32_e32 v6, 0x46004500
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_add_co_u32 v2, s0, s8, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v3, s0, s9, 0, s0
-; GFX1013-NEXT: v_add_co_u32 v4, s0, s10, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v5, s0, s11, 0, s0
-; GFX1013-NEXT: flat_load_dword v0, v[2:3]
-; GFX1013-NEXT: flat_load_dword v1, v[4:5]
+; GFX1013-NEXT: v_mov_b32_e32 v0, s8
+; GFX1013-NEXT: v_mov_b32_e32 v1, s9
+; GFX1013-NEXT: v_mov_b32_e32 v2, s10
+; GFX1013-NEXT: v_mov_b32_e32 v3, s11
+; GFX1013-NEXT: v_add_co_u32 v4, vcc_lo, v0, v6
+; GFX1013-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
+; GFX1013-NEXT: v_add_co_u32 v2, vcc_lo, v2, v6
+; GFX1013-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX1013-NEXT: v_mov_b32_e32 v6, 0x46004500
+; GFX1013-NEXT: flat_load_dword v0, v[4:5]
+; GFX1013-NEXT: flat_load_dword v1, v[2:3]
; GFX1013-NEXT: v_mov_b32_e32 v2, 0
-; GFX1013-NEXT: v_mov_b32_e32 v5, 0x44004200
-; GFX1013-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1013-NEXT: v_mov_b32_e32 v3, 1.0
+; GFX1013-NEXT: v_mov_b32_e32 v4, 2.0
+; GFX1013-NEXT: v_mov_b32_e32 v5, 0x44004200
; GFX1013-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1013-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[12:15] a16
; GFX1013-NEXT: s_waitcnt vmcnt(0)
@@ -1026,19 +1074,21 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1030-NEXT: s_clause 0x1
; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1030-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX1030-NEXT: v_mov_b32_e32 v3, 0
-; GFX1030-NEXT: v_mov_b32_e32 v8, 0x48004700
-; GFX1030-NEXT: v_mov_b32_e32 v7, 0x46004500
-; GFX1030-NEXT: v_mov_b32_e32 v6, 0x44004200
-; GFX1030-NEXT: v_mov_b32_e32 v5, 2.0
; GFX1030-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX1030-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX1030-NEXT: v_mov_b32_e32 v6, 0x44004200
+; GFX1030-NEXT: v_mov_b32_e32 v7, 0x46004500
+; GFX1030-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
+; GFX1030-NEXT: v_mov_b32_e32 v0, s6
+; GFX1030-NEXT: v_mov_b32_e32 v1, s7
+; GFX1030-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1030-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1030-NEXT: flat_load_dword v2, v[0:1]
-; GFX1030-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1030-NEXT: v_mov_b32_e32 v0, 0xb36211c6
+; GFX1030-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1030-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1030-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[0:3] a16
; GFX1030-NEXT: s_waitcnt vmcnt(0)
@@ -1050,19 +1100,21 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-NEXT: s_clause 0x1
; GFX1013-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1013-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1013-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1013-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX1013-NEXT: v_mov_b32_e32 v3, 0
-; GFX1013-NEXT: v_mov_b32_e32 v8, 0x48004700
-; GFX1013-NEXT: v_mov_b32_e32 v7, 0x46004500
-; GFX1013-NEXT: v_mov_b32_e32 v6, 0x44004200
-; GFX1013-NEXT: v_mov_b32_e32 v5, 2.0
; GFX1013-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX1013-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX1013-NEXT: v_mov_b32_e32 v6, 0x44004200
+; GFX1013-NEXT: v_mov_b32_e32 v7, 0x46004500
+; GFX1013-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1013-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX1013-NEXT: v_add_co_ci_u32_e64 v1, s4, s7, 0, s4
+; GFX1013-NEXT: v_mov_b32_e32 v0, s6
+; GFX1013-NEXT: v_mov_b32_e32 v1, s7
+; GFX1013-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1013-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1013-NEXT: flat_load_dword v2, v[0:1]
-; GFX1013-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1013-NEXT: v_mov_b32_e32 v0, 0xb36211c6
+; GFX1013-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1013-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1013-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[0:3] a16
; GFX1013-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
index 255168e17fbd9..28d1241adb67a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.buffer.load.format.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
; Natural mapping
@@ -73,28 +73,26 @@ define amdgpu_ps <2 x half> @raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffs
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY10]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: raw_buffer_load_format_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -144,37 +142,36 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffs
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY13]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY15]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; GFX12-LABEL: name: raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -379,37 +376,36 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffs
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 4095, 0, 0 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY13]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY15]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; GFX12-LABEL: name: raw_buffer_load_format_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset_voffset_add_4095
@@ -507,28 +503,26 @@ define amdgpu_ps <2 x half> @raw_buffer_load_format_v2f16__sgpr_rsrc__sgpr_voffs
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr7
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY7]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:sreg_32(s32) = G_AND [[AMDGPU_READANYLANE]], [[C1]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE1]], [[C1]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:sreg_32(s32) = G_SHL [[AND1]], [[C2]](s32)
- ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(i32) = S_OR_B32 [[AND]](s32), [[SHL]](s32), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32(<2 x f16>) = COPY [[S_OR_B32_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY8]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_1]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_AND_B32_1]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_AND_B32_]], [[S_LSHL_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[S_OR_B32_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; GFX12-LABEL: name: raw_buffer_load_format_v2f16__sgpr_rsrc__sgpr_voffset__sgpr_soffset
@@ -574,36 +568,35 @@ define amdgpu_ps <4 x half> @raw_buffer_load_format_v4f16__sgpr_rsrc__sgpr_voffs
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr7
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY7]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE2:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV2]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE3:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV3]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE]], [[C1]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE1]], [[C1]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[AND1]], [[C2]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:sreg_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:sreg_32(s32) = G_AND [[AMDGPU_READANYLANE2]], [[C1]]
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE3]], [[C1]]
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:sreg_32(s32) = G_SHL [[AND3]], [[C2]](s32)
- ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(i32) = S_OR_B32 [[AND2]](s32), [[SHL1]](s32), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32(<2 x f16>) = COPY [[S_OR_B32_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY8]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_1]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_AND_B32_1]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_AND_B32_]], [[S_LSHL_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_2]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_3]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_AND_B32_3]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_AND_B32_2]], [[S_LSHL_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[S_OR_B32_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[S_OR_B32_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; GFX12-LABEL: name: raw_buffer_load_format_v4f16__sgpr_rsrc__sgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll
index 0fd26208059c7..ab622e0945ea9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.load.format.f16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; Natural mapping
define amdgpu_ps half @raw_ptr_buffer_load_format_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
@@ -57,28 +57,26 @@ define amdgpu_ps <2 x half> @raw_ptr_buffer_load_format_v2f16__sgpr_rsrc__vgpr_v
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY10]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call <2 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v2f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <2 x half> %val
@@ -113,37 +111,36 @@ define amdgpu_ps <4 x half> @raw_ptr_buffer_load_format_v4f16__sgpr_rsrc__vgpr_v
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 0, 0 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY13]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY15]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
ret <4 x half> %val
@@ -279,37 +276,36 @@ define amdgpu_ps <4 x half> @raw_ptr_buffer_load_format_v4f16__sgpr_rsrc__vgpr_v
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 4095, 0, 0 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 4095, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY13]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY15]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%voffset = add i32 %voffset.base, 4095
%val = call <4 x half> @llvm.amdgcn.raw.ptr.buffer.load.format.v4f16(ptr addrspace(8) %rsrc, i32 %voffset, i32 %soffset, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll
index 1b50cc6ebaa12..25dbaf26d59db 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.tbuffer.load.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps half @raw_tbuffer_load_f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %voffset, i32 inreg %soffset) {
@@ -41,28 +41,26 @@ define amdgpu_ps <2 x half> @raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 78, 0, 0 :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY10]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -94,37 +92,36 @@ define amdgpu_ps <4 x half> @raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 78, 0, 0 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY13]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY15]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
index 6dd8bfa569f29..358a570c1dc15 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.tbuffer.load.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -57,28 +57,26 @@ define amdgpu_ps <2 x half> @raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 78, 0, 0 :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY10]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: raw_tbuffer_load_v2f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -125,37 +123,36 @@ define amdgpu_ps <4 x half> @raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY4]], [[COPY5]], 0, 78, 0, 0 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C2]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY4]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY6]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY12]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY13]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY15]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: raw_tbuffer_load_v4f16__sgpr_rsrc__vgpr_voffset__sgpr_soffset
@@ -607,28 +604,26 @@ define amdgpu_ps <2 x half> @raw_tbuffer_load_v2f16__sgpr_rsrc__sgpr_voffset__sg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr7
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY7]], [[COPY5]], 0, 78, 0, 0 :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:sreg_32(s32) = G_AND [[AMDGPU_READANYLANE]], [[C1]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE1]], [[C1]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:sreg_32(s32) = G_SHL [[AND1]], [[C2]](s32)
- ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(i32) = S_OR_B32 [[AND]](s32), [[SHL]](s32), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32(<2 x f16>) = COPY [[S_OR_B32_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY8]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_1]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_AND_B32_1]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_AND_B32_]], [[S_LSHL_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[S_OR_B32_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: raw_tbuffer_load_v2f16__sgpr_rsrc__sgpr_voffset__sgpr_soffset
@@ -671,36 +666,35 @@ define amdgpu_ps <4 x half> @raw_tbuffer_load_v4f16__sgpr_rsrc__sgpr_voffset__sg
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $sgpr7
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr7
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(i32) = G_CONSTANT i32 0
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY [[C]](i32)
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY6]](i32), [[COPY7]], [[COPY5]], 0, 78, 0, 0 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE1:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV1]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE2:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV2]]
- ; UNPACKED-NEXT: [[AMDGPU_READANYLANE3:%[0-9]+]]:sgpr(i32) = G_AMDGPU_READANYLANE [[UV3]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE]], [[C1]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE1]], [[C1]]
- ; UNPACKED-NEXT: [[C2:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:sgpr(s32) = G_SHL [[AND1]], [[C2]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:sgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:sreg_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:sreg_32(s32) = G_AND [[AMDGPU_READANYLANE2]], [[C1]]
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:sgpr(s32) = G_AND [[AMDGPU_READANYLANE3]], [[C1]]
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:sreg_32(s32) = G_SHL [[AND3]], [[C2]](s32)
- ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32(i32) = S_OR_B32 [[AND2]](s32), [[SHL1]](s32), implicit-def dead $scc
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:sreg_32(<2 x f16>) = COPY [[S_OR_B32_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY8]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr7
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN [[COPY6]], [[REG_SEQUENCE]], [[COPY5]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub2
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_OFFEN]].sub3
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY7]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY8]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_1]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_AND_B32_1]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_AND_B32_]], [[S_LSHL_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_2:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_2]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_B32_3:%[0-9]+]]:sreg_32 = S_AND_B32 [[V_READFIRSTLANE_B32_3]], [[S_MOV_B32_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_LSHL_B32_1:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_AND_B32_3]], [[S_MOV_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_OR_B32_1:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_AND_B32_2]], [[S_LSHL_B32_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[S_OR_B32_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[S_OR_B32_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: raw_tbuffer_load_v4f16__sgpr_rsrc__sgpr_voffset__sgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll
index 3b09569912606..12f20e0a6f34c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.sdot4.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.06 < %s | FileCheck --check-prefix=GFX906 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.11 < %s | FileCheck --check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.12 < %s | FileCheck --check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.06 < %s | FileCheck --check-prefix=GFX906 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.11 < %s | FileCheck --check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.12 < %s | FileCheck --check-prefix=GFX10 %s
define i32 @v_sdot4(i32 %a, i32 %b, i32 %c) {
; GFX906-LABEL: v_sdot4:
@@ -41,26 +41,44 @@ define i32 @v_sdot4_cast_v4i8(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX906-LABEL: v_sdot4_cast_v4i8:
; GFX906: ; %bb.0:
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX906-NEXT: v_perm_b32 v4, v4, v5, s4
-; GFX906-NEXT: v_perm_b32 v5, v6, v7, s4
-; GFX906-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX906-NEXT: v_perm_b32 v1, v2, v3, s4
-; GFX906-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX906-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX906-NEXT: v_dot4_i32_i8 v0, v0, v4, v8
+; GFX906-NEXT: v_mov_b32_e32 v10, 8
+; GFX906-NEXT: v_mov_b32_e32 v9, 0xff
+; GFX906-NEXT: v_lshlrev_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT: v_and_or_b32 v0, v0, v9, v1
+; GFX906-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX906-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX906-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX906-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX906-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX906-NEXT: v_lshlrev_b32_sdwa v1, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GFX906-NEXT: v_and_b32_e32 v3, 0xff, v7
+; GFX906-NEXT: v_and_or_b32 v1, v4, v9, v1
+; GFX906-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX906-NEXT: v_or3_b32 v1, v1, v2, v3
+; GFX906-NEXT: v_dot4_i32_i8 v0, v0, v1, v8
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sdot4_cast_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX10-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX10-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10-NEXT: v_dot4c_i32_i8 v8, v0, v2
+; GFX10-NEXT: v_mov_b32_e32 v9, 8
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_and_b32_e32 v5, 0xff, v6
+; GFX10-NEXT: v_and_b32_e32 v6, 0xff, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX10-NEXT: v_and_or_b32 v3, 0xff, v4, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 24, v6
+; GFX10-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT: v_or3_b32 v1, v3, v4, v5
+; GFX10-NEXT: v_dot4c_i32_i8 v8, v0, v1
; GFX10-NEXT: v_mov_b32_e32 v0, v8
; GFX10-NEXT: s_setpc_b64 s[30:31]
%a.cast = bitcast <4 x i8> %a to i32
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
index 16dd85ca01bf4..3068a971f0cd7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
@@ -211,17 +211,21 @@ define amdgpu_kernel void @set_inactive_v2i16(ptr addrspace(1) %out, <2 x i16> %
define amdgpu_kernel void @set_inactive_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
; GCN-LABEL: set_inactive_v2f16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_movk_i32 s3, 0x3c00
+; GCN-NEXT: s_bfe_u32 s3, s3, 0x100000
+; GCN-NEXT: s_lshl_b32 s4, s3, 16
+; GCN-NEXT: s_or_b32 s4, s3, s4
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_mov_b32_e32 v0, 0x3c003c00
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[2:3]
+; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: v_mov_b32_e32 v1, v0
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x half> @llvm.amdgcn.set.inactive.v2f16(<2 x half> %in, <2 x half> <half 1.0, half 1.0>) #0
@@ -279,17 +283,21 @@ define amdgpu_kernel void @set_inactive_v2f32(ptr addrspace(1) %out, <2 x float>
define amdgpu_kernel void @set_inactive_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) {
; GCN-LABEL: set_inactive_v2bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_movk_i32 s3, 0x3f80
+; GCN-NEXT: s_bfe_u32 s3, s3, 0x100000
+; GCN-NEXT: s_lshl_b32 s4, s3, 16
+; GCN-NEXT: s_or_b32 s4, s3, s4
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_mov_b32_e32 v0, 0x3f803f80
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[2:3]
+; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: v_mov_b32_e32 v1, v0
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x bfloat> @llvm.amdgcn.set.inactive.v2bf16(<2 x bfloat> %in, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) #0
@@ -328,23 +336,25 @@ define amdgpu_kernel void @set_inactive_v4f16(ptr addrspace(1) %out, <4 x half>
; GCN-LABEL: set_inactive_v4f16:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_movk_i32 s4, 0x3c00
+; GCN-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GCN-NEXT: s_lshl_b32 s5, s4, 16
+; GCN-NEXT: s_or_b32 s6, s4, s5
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GCN-NEXT: v_mov_b32_e32 v0, 0x3c003c00
-; GCN-NEXT: v_cndmask_b32_e64 v1, v0, v2, s[0:1]
-; GCN-NEXT: s_mov_b64 exec, s[0:1]
-; GCN-NEXT: v_mov_b32_e32 v2, s2
-; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT: s_mov_b64 exec, s[0:1]
-; GCN-NEXT: v_mov_b32_e32 v2, v0
+; GCN-NEXT: v_mov_b32_e32 v3, s2
+; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_cndmask_b32_e64 v1, v0, v3, s[4:5]
+; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GCN-NEXT: v_cndmask_b32_e64 v2, v0, v3, s[2:3]
+; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: v_mov_b32_e32 v3, v1
-; GCN-NEXT: buffer_store_dwordx2 v[2:3], off, s[4:7], 0
+; GCN-NEXT: v_mov_b32_e32 v4, v2
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: buffer_store_dwordx2 v[3:4], off, s[0:3], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <4 x half> @llvm.amdgcn.set.inactive.v4f16(<4 x half> %in, <4 x half> <half 1.0, half 1.0, half 1.0, half 1.0>) #0
%tmp = call <4 x half> @llvm.amdgcn.strict.wwm.v4f16(<4 x half> %tmp.0)
@@ -356,23 +366,25 @@ define amdgpu_kernel void @set_inactive_v4bf16(ptr addrspace(1) %out, <4 x bfloa
; GCN-LABEL: set_inactive_v4bf16:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_movk_i32 s4, 0x3f80
+; GCN-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GCN-NEXT: s_lshl_b32 s5, s4, 16
+; GCN-NEXT: s_or_b32 s6, s4, s5
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GCN-NEXT: v_mov_b32_e32 v0, 0x3f803f80
-; GCN-NEXT: v_cndmask_b32_e64 v1, v0, v2, s[0:1]
-; GCN-NEXT: s_mov_b64 exec, s[0:1]
-; GCN-NEXT: v_mov_b32_e32 v2, s2
-; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
-; GCN-NEXT: s_mov_b64 exec, s[0:1]
-; GCN-NEXT: v_mov_b32_e32 v2, v0
+; GCN-NEXT: v_mov_b32_e32 v3, s2
+; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: v_cndmask_b32_e64 v1, v0, v3, s[4:5]
+; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GCN-NEXT: v_cndmask_b32_e64 v2, v0, v3, s[2:3]
+; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: v_mov_b32_e32 v3, v1
-; GCN-NEXT: buffer_store_dwordx2 v[2:3], off, s[4:7], 0
+; GCN-NEXT: v_mov_b32_e32 v4, v2
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: buffer_store_dwordx2 v[3:4], off, s[0:3], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <4 x bfloat> @llvm.amdgcn.set.inactive.v4bf16(<4 x bfloat> %in, <4 x bfloat> <bfloat 1.0, bfloat 1.0, bfloat 1.0, bfloat 1.0>) #0
%tmp = call <4 x bfloat> @llvm.amdgcn.strict.wwm.v4bf16(<4 x bfloat> %tmp.0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
index 2ba129f01b740..1588f254b801d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.buffer.load.format.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=GFX12 %s
@@ -63,27 +63,28 @@ define amdgpu_ps <2 x half> @struct_buffer_load_format_v2f16__sgpr_rsrc__vgpr_vi
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: struct_buffer_load_format_v2f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -134,36 +135,38 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__sgpr_rsrc__vgpr_vi
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY13]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY10]], [[COPY15]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY16]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: struct_buffer_load_format_v4f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -216,42 +219,43 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__vpr_rsrc__sgpr_vin
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
- ; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR1]](<4 x i32>), [[COPY7]](i32), [[COPY8]], [[INTRINSIC_CONVERGENT4]], 0, 0, -1 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
@@ -261,27 +265,28 @@ define amdgpu_ps <4 x half> @struct_buffer_load_format_v4f16__vpr_rsrc__sgpr_vin
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:vgpr(i32), [[UV5:%[0-9]+]]:vgpr(i32), [[UV6:%[0-9]+]]:vgpr(i32), [[UV7:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr(s32) = G_AND [[UV4]], [[COPY9]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV5]], [[COPY10]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND3]], [[COPY11]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND2]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND4:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV6]], [[COPY12]]
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND5:%[0-9]+]]:vgpr(s32) = G_AND [[UV7]], [[COPY13]]
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND5]], [[COPY14]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND4]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY15]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY13]], [[COPY17]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY14]], [[COPY18]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY19]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY15]], [[COPY20]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY16]], [[COPY21]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY22]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: struct_buffer_load_format_v4f16__vpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
index 4e98a9d52ed84..1f3f54518683f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.load.format.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=UNPACKED %s
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefix=PACKED %s
define amdgpu_ps half @struct_ptr_buffer_load_format_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
@@ -45,27 +45,28 @@ define amdgpu_ps <2 x half> @struct_ptr_buffer_load_format_v2f16__sgpr_rsrc__vgp
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
;
; PACKED-LABEL: name: struct_ptr_buffer_load_format_v2f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -99,36 +100,38 @@ define amdgpu_ps <4 x half> @struct_ptr_buffer_load_format_v4f16__sgpr_rsrc__vgp
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 0, -1 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY13]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY10]], [[COPY15]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY16]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: struct_ptr_buffer_load_format_v4f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -161,42 +164,43 @@ define amdgpu_ps <4 x half> @struct_ptr_buffer_load_format_v4f16__vpr_rsrc__sgpr
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
- ; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[AMDGPU_BUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_BUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR1]](<4 x i32>), [[COPY7]](i32), [[COPY8]], [[INTRINSIC_CONVERGENT4]], 0, 0, -1 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; UNPACKED-NEXT: [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
@@ -206,27 +210,28 @@ define amdgpu_ps <4 x half> @struct_ptr_buffer_load_format_v4f16__vpr_rsrc__sgpr
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:vgpr(i32), [[UV5:%[0-9]+]]:vgpr(i32), [[UV6:%[0-9]+]]:vgpr(i32), [[UV7:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_BUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr(s32) = G_AND [[UV4]], [[COPY9]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV5]], [[COPY10]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND3]], [[COPY11]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND2]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND4:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV6]], [[COPY12]]
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND5:%[0-9]+]]:vgpr(s32) = G_AND [[UV7]], [[COPY13]]
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND5]], [[COPY14]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND4]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY15]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[BUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY13]], [[COPY17]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY14]], [[COPY18]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY19]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY15]], [[COPY20]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY16]], [[COPY21]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY22]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
;
; PACKED-LABEL: name: struct_ptr_buffer_load_format_v4f16__vpr_rsrc__sgpr_vindex__sgpr_voffset__vgpr_soffset
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
index d327664e18032..f418ec0d41c55 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.tbuffer.load.f16.ll
@@ -2,7 +2,7 @@
; RUN: llc -global-isel -mtriple=amdgpu8.10-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -mattr=+wavefrontsize64 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+wavefrontsize64 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=PACKED %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(ptr addrspace(8) inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -64,27 +64,28 @@ define amdgpu_ps <2 x half> @struct_tbuffer_load_v2f16__sgpr_rsrc__vgpr_vindex__
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, -1 :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call <2 x half> @llvm.amdgcn.struct.ptr.tbuffer.load.v2f16(ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret <2 x half> %val
@@ -121,36 +122,38 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, -1 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY13]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY10]], [[COPY15]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY16]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.struct.ptr.tbuffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret <4 x half> %val
@@ -259,42 +262,43 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
- ; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR1]](<4 x i32>), [[COPY7]](i32), [[COPY8]], [[INTRINSIC_CONVERGENT4]], 0, 78, 0, -1 :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>) from %ir.rsrc, align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
@@ -304,27 +308,28 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:vgpr(i32), [[UV5:%[0-9]+]]:vgpr(i32), [[UV6:%[0-9]+]]:vgpr(i32), [[UV7:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr(s32) = G_AND [[UV4]], [[COPY9]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV5]], [[COPY10]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND3]], [[COPY11]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND2]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND4:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV6]], [[COPY12]]
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND5:%[0-9]+]]:vgpr(s32) = G_AND [[UV7]], [[COPY13]]
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND5]], [[COPY14]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND4]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY15]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY13]], [[COPY17]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY14]], [[COPY18]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY19]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY15]], [[COPY20]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY16]], [[COPY21]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY22]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.struct.ptr.tbuffer.load.v4f16(ptr addrspace(8) %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret <4 x half> %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
index 2c164e3be1578..a568c9b324320 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.tbuffer.load.f16.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d -mattr=+wavefrontsize64 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+wavefrontsize64 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=PACKED %s
; RUN: llc -global-isel -mtriple=amdgpu12.00-mesa-mesa3d -mattr=+wavefrontsize64 -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=GFX12 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d -stop-after=instruction-select -o - %s | FileCheck -check-prefixes=UNPACKED %s
define amdgpu_ps half @struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset(<4 x i32> inreg %rsrc, i32 %vindex, i32 %voffset, i32 inreg %soffset) {
; PACKED-LABEL: name: struct_tbuffer_load_f16__sgpr_rsrc__vgpr_vindex__vgpr_voffset__sgpr_soffset
@@ -99,27 +99,28 @@ define amdgpu_ps <2 x half> @struct_tbuffer_load_v2f16__sgpr_rsrc__vgpr_vindex__
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<2 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, -1 :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<2 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND]](s32), [[SHL]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[COPY10]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN:%[0-9]+]]:vreg_64 = TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<2 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XY_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY11]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0
%val = call <2 x half> @llvm.amdgcn.struct.tbuffer.load.v2f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret <2 x half> %val
@@ -176,36 +177,38 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__sgpr_rsrc__vgpr_vindex__
; UNPACKED: bb.1 (%ir-block.0):
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $sgpr4, $sgpr5, $sgpr6, $vgpr0, $vgpr1
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sgpr(i32) = COPY $sgpr4
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sgpr(i32) = COPY $sgpr5
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sgpr(i32) = COPY $sgpr6
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR]](<4 x i32>), [[COPY4]](i32), [[COPY5]], [[COPY6]], 0, 78, 0, -1 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i32), [[UV1:%[0-9]+]]:vgpr(i32), [[UV2:%[0-9]+]]:vgpr(i32), [[UV3:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vgpr(s32) = G_AND [[UV]], [[COPY7]]
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vgpr(s32) = G_AND [[UV1]], [[COPY8]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND1]], [[COPY9]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV2]], [[COPY10]]
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV3]], [[COPY11]]
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND3]], [[COPY12]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND2]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY13]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY $sgpr4
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY $sgpr5
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY $sgpr6
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY4]], %subreg.sub0, [[COPY5]], %subreg.sub1
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE]], [[COPY6]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY7]], [[COPY11]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY8]], [[COPY12]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY13]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY9]], [[COPY14]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY10]], [[COPY15]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY16]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.struct.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret <4 x half> %val
@@ -389,42 +392,43 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; UNPACKED-NEXT: successors: %bb.2(0x80000000)
; UNPACKED-NEXT: liveins: $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr(i32) = COPY $vgpr0
- ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr(i32) = COPY $vgpr1
- ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr(i32) = COPY $vgpr2
- ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr(i32) = COPY $vgpr3
- ; UNPACKED-NEXT: [[BUILD_VECTOR:%[0-9]+]]:vgpr(<4 x i32>) = G_BUILD_VECTOR [[COPY]](i32), [[COPY1]](i32), [[COPY2]](i32), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sgpr(i32) = COPY $sgpr2
- ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sgpr(i32) = COPY $sgpr3
- ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr(i32) = COPY $vgpr4
- ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr(i32) = COPY [[COPY4]](i32)
- ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr(i32) = COPY [[COPY5]](i32)
- ; UNPACKED-NEXT: [[DEF:%[0-9]+]]:sreg_64_xexec = IMPLICIT_DEF
+ ; UNPACKED-NEXT: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+ ; UNPACKED-NEXT: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+ ; UNPACKED-NEXT: [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+ ; UNPACKED-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY $vgpr3
+ ; UNPACKED-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vreg_128 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2, [[COPY3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY $sgpr2
+ ; UNPACKED-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY $sgpr3
+ ; UNPACKED-NEXT: [[COPY6:%[0-9]+]]:vgpr_32 = COPY $vgpr4
+ ; UNPACKED-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[COPY4]]
+ ; UNPACKED-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[COPY5]]
; UNPACKED-NEXT: [[S_MOV_B64_:%[0-9]+]]:sreg_64_xexec = S_MOV_B64 $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.2:
; UNPACKED-NEXT: successors: %bb.3(0x80000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY1]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY2]](i32)
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT3:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY3]](i32)
- ; UNPACKED-NEXT: [[BUILD_VECTOR1:%[0-9]+]]:sgpr(<4 x i32>) = G_BUILD_VECTOR [[INTRINSIC_CONVERGENT]](i32), [[INTRINSIC_CONVERGENT1]](i32), [[INTRINSIC_CONVERGENT2]](i32), [[INTRINSIC_CONVERGENT3]](i32)
- ; UNPACKED-NEXT: [[UV:%[0-9]+]]:vgpr(i64), [[UV1:%[0-9]+]]:vgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR]](<4 x i32>)
- ; UNPACKED-NEXT: [[UV2:%[0-9]+]]:sgpr(i64), [[UV3:%[0-9]+]]:sgpr(i64) = G_UNMERGE_VALUES [[BUILD_VECTOR1]](<4 x i32>)
- ; UNPACKED-NEXT: [[ICMP:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV2]](i64), [[UV]]
- ; UNPACKED-NEXT: [[ICMP1:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[UV3]](i64), [[UV1]]
- ; UNPACKED-NEXT: [[AND:%[0-9]+]]:vcc(s1) = G_AND [[ICMP]], [[ICMP1]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT4:%[0-9]+]]:sgpr(i32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[COPY6]](i32)
- ; UNPACKED-NEXT: [[ICMP2:%[0-9]+]]:vcc(s1) = G_ICMP intpred(eq), [[INTRINSIC_CONVERGENT4]](i32), [[COPY6]]
- ; UNPACKED-NEXT: [[AND1:%[0-9]+]]:vcc(s1) = G_AND [[AND]], [[ICMP2]]
- ; UNPACKED-NEXT: [[INTRINSIC_CONVERGENT5:%[0-9]+]]:sreg_64_xexec(i64) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.ballot), [[AND1]](s1)
- ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[INTRINSIC_CONVERGENT5]](i64), implicit-def $exec, implicit-def $scc, implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_1:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_2:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY2]], implicit $exec
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_3:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY3]], implicit $exec
+ ; UNPACKED-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE [[V_READFIRSTLANE_B32_]], %subreg.sub0, [[V_READFIRSTLANE_B32_1]], %subreg.sub1, [[V_READFIRSTLANE_B32_2]], %subreg.sub2, [[V_READFIRSTLANE_B32_3]], %subreg.sub3
+ ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vreg_64 = COPY [[REG_SEQUENCE]].sub2_sub3
+ ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub0_sub1
+ ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE1]].sub2_sub3
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY11]], [[COPY9]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U64_e64_1:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U64_e64 [[COPY12]], [[COPY10]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[V_CMP_EQ_U64_e64_]], [[V_CMP_EQ_U64_e64_1]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[V_READFIRSTLANE_B32_4:%[0-9]+]]:sreg_32_xm0 = V_READFIRSTLANE_B32 [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[V_CMP_EQ_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_EQ_U32_e64 [[V_READFIRSTLANE_B32_4]], [[COPY6]], implicit $exec
+ ; UNPACKED-NEXT: [[S_AND_B64_1:%[0-9]+]]:sreg_64_xexec = S_AND_B64 [[S_AND_B64_]], [[V_CMP_EQ_U32_e64_]], implicit-def dead $scc
+ ; UNPACKED-NEXT: [[S_AND_SAVEEXEC_B64_:%[0-9]+]]:sreg_64_xexec = S_AND_SAVEEXEC_B64 killed [[S_AND_B64_1]], implicit-def $exec, implicit-def $scc, implicit $exec
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.3:
; UNPACKED-NEXT: successors: %bb.4(0x40000000), %bb.2(0x40000000)
; UNPACKED-NEXT: {{ $}}
- ; UNPACKED-NEXT: [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_:%[0-9]+]]:vgpr(<4 x i32>) = G_AMDGPU_TBUFFER_LOAD_FORMAT_D16 [[BUILD_VECTOR1]](<4 x i32>), [[COPY7]](i32), [[COPY8]], [[INTRINSIC_CONVERGENT4]], 0, 78, 0, -1 :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
+ ; UNPACKED-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY7]], %subreg.sub0, [[COPY8]], %subreg.sub1
+ ; UNPACKED-NEXT: [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN:%[0-9]+]]:vreg_128 = TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[V_READFIRSTLANE_B32_4]], 0, 78, 0, 0, implicit $exec :: (dereferenceable load (<4 x f16>), align 1, addrspace 8)
; UNPACKED-NEXT: $exec = S_XOR_B64_term $exec, [[S_AND_SAVEEXEC_B64_]], implicit-def $scc
; UNPACKED-NEXT: SI_WATERFALL_LOOP %bb.2, implicit $exec
; UNPACKED-NEXT: {{ $}}
@@ -434,27 +438,28 @@ define amdgpu_ps <4 x half> @struct_tbuffer_load_v4f16__vgpr_rsrc__sgpr_vindex__
; UNPACKED-NEXT: $exec = S_MOV_B64_term [[S_MOV_B64_]]
; UNPACKED-NEXT: {{ $}}
; UNPACKED-NEXT: bb.5:
- ; UNPACKED-NEXT: [[UV4:%[0-9]+]]:vgpr(i32), [[UV5:%[0-9]+]]:vgpr(i32), [[UV6:%[0-9]+]]:vgpr(i32), [[UV7:%[0-9]+]]:vgpr(i32) = G_UNMERGE_VALUES [[AMDGPU_TBUFFER_LOAD_FORMAT_D16_]](<4 x i32>)
- ; UNPACKED-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 65535
- ; UNPACKED-NEXT: [[COPY9:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND2:%[0-9]+]]:vgpr(s32) = G_AND [[UV4]], [[COPY9]]
- ; UNPACKED-NEXT: [[COPY10:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND3:%[0-9]+]]:vgpr(s32) = G_AND [[UV5]], [[COPY10]]
- ; UNPACKED-NEXT: [[C1:%[0-9]+]]:sgpr(s32) = G_CONSTANT i32 16
- ; UNPACKED-NEXT: [[COPY11:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL:%[0-9]+]]:vgpr(s32) = G_SHL [[AND3]], [[COPY11]](s32)
- ; UNPACKED-NEXT: [[OR:%[0-9]+]]:vgpr(i32) = G_OR [[AND2]], [[SHL]]
- ; UNPACKED-NEXT: [[BITCAST:%[0-9]+]]:vgpr_32(<2 x f16>) = G_BITCAST [[OR]](i32)
- ; UNPACKED-NEXT: [[COPY12:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND4:%[0-9]+]]:vgpr_32(s32) = G_AND [[UV6]], [[COPY12]]
- ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
- ; UNPACKED-NEXT: [[AND5:%[0-9]+]]:vgpr(s32) = G_AND [[UV7]], [[COPY13]]
- ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
- ; UNPACKED-NEXT: [[SHL1:%[0-9]+]]:vgpr_32(s32) = G_SHL [[AND5]], [[COPY14]](s32)
- ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32(i32) = V_OR_B32_e64 [[AND4]](s32), [[SHL1]](s32), implicit $exec
- ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32(<2 x f16>) = COPY [[V_OR_B32_e64_]](i32)
- ; UNPACKED-NEXT: $vgpr0 = COPY [[BITCAST]](<2 x f16>)
- ; UNPACKED-NEXT: $vgpr1 = COPY [[COPY15]](<2 x f16>)
+ ; UNPACKED-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub0
+ ; UNPACKED-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub1
+ ; UNPACKED-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub2
+ ; UNPACKED-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[TBUFFER_LOAD_FORMAT_D16_XYZW_gfx80_BOTHEN]].sub3
+ ; UNPACKED-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 65535
+ ; UNPACKED-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY13]], [[COPY17]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY14]], [[COPY18]], implicit $exec
+ ; UNPACKED-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 16
+ ; UNPACKED-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY19]], [[V_AND_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_]], [[V_LSHLREV_B32_e64_]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY15]], [[COPY20]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; UNPACKED-NEXT: [[V_AND_B32_e64_3:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY16]], [[COPY21]], implicit $exec
+ ; UNPACKED-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_1]]
+ ; UNPACKED-NEXT: [[V_LSHLREV_B32_e64_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e64 [[COPY22]], [[V_AND_B32_e64_3]], implicit $exec
+ ; UNPACKED-NEXT: [[V_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e64 [[V_AND_B32_e64_2]], [[V_LSHLREV_B32_e64_1]], implicit $exec
+ ; UNPACKED-NEXT: $vgpr0 = COPY [[V_OR_B32_e64_]]
+ ; UNPACKED-NEXT: $vgpr1 = COPY [[V_OR_B32_e64_1]]
; UNPACKED-NEXT: SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
%val = call <4 x half> @llvm.amdgcn.struct.tbuffer.load.v4f16(<4 x i32> %rsrc, i32 %vindex, i32 %voffset, i32 %soffset, i32 78, i32 0)
ret <4 x half> %val
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll
index 321551651c0e3..2a4502cc45868 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.udot4.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.06 < %s | FileCheck --check-prefix=GFX906 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.11 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.12 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.06 < %s | FileCheck --check-prefix=GFX906 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.11 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.12 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -amdgpu-enable-delay-alu=0 < %s | FileCheck --check-prefixes=GFX10PLUS,GFX11 %s
define i32 @v_udot4(i32 %a, i32 %b, i32 %c) {
; GFX906-LABEL: v_udot4:
@@ -41,27 +41,67 @@ define i32 @v_udot4_cast_v4i8(<4 x i8> %a, <4 x i8> %b, i32 %c) {
; GFX906-LABEL: v_udot4_cast_v4i8:
; GFX906: ; %bb.0:
; GFX906-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX906-NEXT: s_mov_b32 s4, 0xc0c0004
-; GFX906-NEXT: v_perm_b32 v4, v4, v5, s4
-; GFX906-NEXT: v_perm_b32 v5, v6, v7, s4
-; GFX906-NEXT: v_perm_b32 v0, v0, v1, s4
-; GFX906-NEXT: v_perm_b32 v1, v2, v3, s4
-; GFX906-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GFX906-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX906-NEXT: v_dot4_u32_u8 v0, v0, v4, v8
+; GFX906-NEXT: v_mov_b32_e32 v10, 8
+; GFX906-NEXT: v_mov_b32_e32 v9, 0xff
+; GFX906-NEXT: v_lshlrev_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT: v_and_or_b32 v0, v0, v9, v1
+; GFX906-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX906-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX906-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX906-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX906-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX906-NEXT: v_lshlrev_b32_sdwa v1, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GFX906-NEXT: v_and_b32_e32 v3, 0xff, v7
+; GFX906-NEXT: v_and_or_b32 v1, v4, v9, v1
+; GFX906-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX906-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX906-NEXT: v_or3_b32 v1, v1, v2, v3
+; GFX906-NEXT: v_dot4_u32_u8 v0, v0, v1, v8
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; GFX10PLUS-LABEL: v_udot4_cast_v4i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10PLUS-NEXT: v_perm_b32 v4, v4, v5, 0xc0c0004
-; GFX10PLUS-NEXT: v_perm_b32 v5, v6, v7, 0xc0c0004
-; GFX10PLUS-NEXT: v_perm_b32 v0, v0, v1, 0xc0c0004
-; GFX10PLUS-NEXT: v_perm_b32 v1, v2, v3, 0xc0c0004
-; GFX10PLUS-NEXT: v_lshl_or_b32 v2, v5, 16, v4
-; GFX10PLUS-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX10PLUS-NEXT: v_dot4_u32_u8 v0, v0, v2, v8
-; GFX10PLUS-NEXT: s_setpc_b64 s[30:31]
+; GFX10-LABEL: v_udot4_cast_v4i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v9, 8
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX10-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX10-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GFX10-NEXT: v_lshlrev_b32_sdwa v3, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX10-NEXT: v_and_b32_e32 v5, 0xff, v6
+; GFX10-NEXT: v_and_b32_e32 v6, 0xff, v7
+; GFX10-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX10-NEXT: v_and_or_b32 v3, 0xff, v4, v3
+; GFX10-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX10-NEXT: v_lshlrev_b32_e32 v5, 24, v6
+; GFX10-NEXT: v_or3_b32 v0, v0, v1, v2
+; GFX10-NEXT: v_or3_b32 v1, v3, v4, v5
+; GFX10-NEXT: v_dot4_u32_u8 v0, v0, v1, v8
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_udot4_cast_v4i8:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-NEXT: v_and_b32_e32 v5, 0xff, v5
+; GFX11-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 8, v5
+; GFX11-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX11-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX11-NEXT: v_and_or_b32 v0, 0xff, v0, v1
+; GFX11-NEXT: v_and_b32_e32 v1, 0xff, v6
+; GFX11-NEXT: v_and_b32_e32 v6, 0xff, v7
+; GFX11-NEXT: v_and_or_b32 v4, 0xff, v4, v5
+; GFX11-NEXT: v_or3_b32 v0, v0, v2, v3
+; GFX11-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-NEXT: v_lshlrev_b32_e32 v5, 24, v6
+; GFX11-NEXT: v_or3_b32 v1, v4, v1, v5
+; GFX11-NEXT: v_dot4_u32_u8 v0, v0, v1, v8
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%a.cast = bitcast <4 x i8> %a to i32
%b.cast = bitcast <4 x i8> %b to i32
%r = call i32 @llvm.amdgcn.udot4(i32 %a.cast, i32 %b.cast, i32 %c, i1 false)
@@ -130,6 +170,3 @@ define amdgpu_ps i32 @v_udot4_s_s_s(i32 inreg %a, i32 inreg %b, i32 inreg %c) {
declare i32 @llvm.amdgcn.udot4(i32, i32, i32, i1 immarg) #0
attributes #0 = { nounwind readnone speculatable }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX10: {{.*}}
-; GFX11: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
index b524812a499f9..02b68822047c3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.demote.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefix=SI %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02 < %s | FileCheck -check-prefix=SI %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefix=GFX10-32 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 -mattr=+wavefrontsize64 < %s | FileCheck -check-prefix=GFX10-64 %s
@@ -666,6 +666,13 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
; SI-NEXT: s_mov_b64 s[0:1], exec
; SI-NEXT: s_wqm_b64 exec, exec
; SI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; SI-NEXT: s_movk_i32 s2, 0x3c00
+; SI-NEXT: s_bfe_u32 s3, 0, 0x100000
+; SI-NEXT: s_bfe_u32 s2, s2, 0x100000
+; SI-NEXT: s_lshl_b32 s4, s3, 16
+; SI-NEXT: s_or_b32 s4, s2, s4
+; SI-NEXT: s_lshl_b32 s2, s2, 16
+; SI-NEXT: s_or_b32 s5, s3, s2
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; SI-NEXT: s_and_saveexec_b64 s[2:3], vcc
; SI-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
@@ -674,24 +681,25 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
; SI-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
; SI-NEXT: s_cbranch_scc0 .LBB6_7
; SI-NEXT: ; %bb.2: ; %.demote0
-; SI-NEXT: s_wqm_b64 s[4:5], s[0:1]
-; SI-NEXT: s_and_b64 exec, exec, s[4:5]
+; SI-NEXT: s_wqm_b64 s[6:7], s[0:1]
+; SI-NEXT: s_and_b64 exec, exec, s[6:7]
; SI-NEXT: .LBB6_3: ; %.continue0
; SI-NEXT: s_or_b64 exec, exec, s[2:3]
-; SI-NEXT: s_mov_b64 s[2:3], s[0:1]
-; SI-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s[2:3]
+; SI-NEXT: s_mov_b64 s[6:7], s[0:1]
+; SI-NEXT: v_cndmask_b32_e64 v0, 1.0, 0, s[6:7]
; SI-NEXT: v_mov_b32_e32 v1, v0
-; SI-NEXT: s_xor_b64 s[2:3], s[0:1], -1
+; SI-NEXT: s_mov_b64 s[2:3], exec
; SI-NEXT: s_nop 0
; SI-NEXT: v_mov_b32_dpp v1, v1 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
; SI-NEXT: s_nop 1
; SI-NEXT: v_subrev_f32_dpp v0, v0, v1 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; SI-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec
; SI-NEXT: s_and_b64 exec, exec, s[0:1]
-; SI-NEXT: v_cmp_neq_f32_e32 vcc, 0, v0
-; SI-NEXT: s_or_b64 s[2:3], s[2:3], vcc
-; SI-NEXT: s_and_saveexec_b64 s[4:5], s[2:3]
-; SI-NEXT: s_xor_b64 s[2:3], exec, s[4:5]
+; SI-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
+; SI-NEXT: s_and_b64 s[6:7], s[0:1], vcc
+; SI-NEXT: s_xor_b64 s[2:3], s[6:7], s[2:3]
+; SI-NEXT: s_and_saveexec_b64 s[6:7], s[2:3]
+; SI-NEXT: s_xor_b64 s[2:3], exec, s[6:7]
; SI-NEXT: s_cbranch_execz .LBB6_6
; SI-NEXT: ; %bb.4: ; %.demote1
; SI-NEXT: s_andn2_b64 s[0:1], s[0:1], exec
@@ -700,9 +708,9 @@ define amdgpu_ps void @wqm_deriv(<2 x float> %input, float %arg, i32 %index) {
; SI-NEXT: s_mov_b64 exec, 0
; SI-NEXT: .LBB6_6: ; %.continue1
; SI-NEXT: s_or_b64 exec, exec, s[2:3]
-; SI-NEXT: v_bfrev_b32_e32 v0, 60
-; SI-NEXT: v_mov_b32_e32 v1, 0x3c00
-; SI-NEXT: exp mrt0, v1, v1, v0, v0 done compr vm
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: exp mrt0, v0, v0, v1, v1 done compr vm
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB6_7:
; SI-NEXT: s_mov_b64 exec, 0
@@ -892,7 +900,14 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; SI-NEXT: s_mov_b64 s[0:1], exec
; SI-NEXT: s_wqm_b64 exec, exec
; SI-NEXT: v_cvt_i32_f32_e32 v0, v0
-; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_movk_i32 s2, 0x3c00
+; SI-NEXT: s_bfe_u32 s3, 0, 0x100000
+; SI-NEXT: s_bfe_u32 s2, s2, 0x100000
+; SI-NEXT: s_lshl_b32 s4, s3, 16
+; SI-NEXT: s_or_b32 s6, s2, s4
+; SI-NEXT: s_lshl_b32 s2, s2, 16
+; SI-NEXT: s_or_b32 s7, s3, s2
+; SI-NEXT: s_mov_b32 s8, 0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; SI-NEXT: s_and_saveexec_b64 s[2:3], vcc
; SI-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
@@ -910,27 +925,28 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; SI-NEXT: .LBB7_4: ; %.continue1
; SI-NEXT: ; in Loop: Header=BB7_5 Depth=1
; SI-NEXT: s_or_b64 exec, exec, s[4:5]
-; SI-NEXT: s_add_i32 s6, s6, 1
-; SI-NEXT: v_cmp_ge_i32_e32 vcc, s6, v1
+; SI-NEXT: s_add_i32 s8, s8, 1
+; SI-NEXT: v_cmp_ge_i32_e32 vcc, s8, v1
; SI-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
; SI-NEXT: s_andn2_b64 exec, exec, s[2:3]
; SI-NEXT: s_cbranch_execz .LBB7_8
; SI-NEXT: .LBB7_5: ; %.continue0
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
-; SI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[4:5]
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: s_mov_b64 s[10:11], s[0:1]
+; SI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[10:11]
; SI-NEXT: v_mov_b32_e32 v2, v0
-; SI-NEXT: s_xor_b64 s[4:5], s[0:1], -1
+; SI-NEXT: s_mov_b64 s[4:5], exec
; SI-NEXT: s_nop 0
; SI-NEXT: v_mov_b32_dpp v2, v2 quad_perm:[1,1,1,1] row_mask:0xf bank_mask:0xf bound_ctrl:1
; SI-NEXT: s_nop 1
; SI-NEXT: v_subrev_f32_dpp v0, v0, v2 quad_perm:[0,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
; SI-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $exec
-; SI-NEXT: v_cmp_neq_f32_e32 vcc, 0, v0
-; SI-NEXT: s_or_b64 s[4:5], s[4:5], vcc
-; SI-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; SI-NEXT: s_xor_b64 s[4:5], exec, s[8:9]
+; SI-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0
+; SI-NEXT: s_and_b64 s[10:11], s[0:1], vcc
+; SI-NEXT: s_xor_b64 s[4:5], s[10:11], s[4:5]
+; SI-NEXT: s_and_saveexec_b64 s[10:11], s[4:5]
+; SI-NEXT: s_xor_b64 s[4:5], exec, s[10:11]
; SI-NEXT: s_cbranch_execz .LBB7_4
; SI-NEXT: ; %bb.6: ; %.demote1
; SI-NEXT: ; in Loop: Header=BB7_5 Depth=1
@@ -938,15 +954,15 @@ define amdgpu_ps void @wqm_deriv_loop(<2 x float> %input, float %arg, i32 %index
; SI-NEXT: s_cbranch_scc0 .LBB7_9
; SI-NEXT: ; %bb.7: ; %.demote1
; SI-NEXT: ; in Loop: Header=BB7_5 Depth=1
-; SI-NEXT: s_wqm_b64 s[8:9], s[0:1]
-; SI-NEXT: s_and_b64 exec, exec, s[8:9]
+; SI-NEXT: s_wqm_b64 s[10:11], s[0:1]
+; SI-NEXT: s_and_b64 exec, exec, s[10:11]
; SI-NEXT: s_branch .LBB7_4
; SI-NEXT: .LBB7_8: ; %.return
; SI-NEXT: s_or_b64 exec, exec, s[2:3]
; SI-NEXT: s_and_b64 exec, exec, s[0:1]
-; SI-NEXT: v_bfrev_b32_e32 v0, 60
-; SI-NEXT: v_mov_b32_e32 v1, 0x3c00
-; SI-NEXT: exp mrt0, v1, v1, v0, v0 done compr vm
+; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v1, s7
+; SI-NEXT: exp mrt0, v0, v0, v1, v1 done compr vm
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB7_9:
; SI-NEXT: s_mov_b64 exec, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
index 576d1f8b521a5..9552181b7d11b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memset.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-- -amdgpu-memcpy-loop-unroll=2 -mem-intrinsic-expand-size=35 %s -o - | FileCheck -check-prefix=LOOP %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-- -amdgpu-memcpy-loop-unroll=2 -mem-intrinsic-expand-size=35 %s -o - | FileCheck -check-prefix=LOOP %s
; RUN: llc -global-isel -mtriple=amdgpu6.00-- -amdgpu-memcpy-loop-unroll=2 -mem-intrinsic-expand-size=37 %s -o - | FileCheck -check-prefix=UNROLL %s
declare void @llvm.memset.p1.i32(ptr addrspace(1), i8, i32, i1)
@@ -7,97 +7,78 @@ declare void @llvm.memset.p1.i32(ptr addrspace(1), i8, i32, i1)
define amdgpu_cs void @memset_p1i8(ptr addrspace(1) %dst, i8 %val) {
; LOOP-LABEL: memset_p1i8:
; LOOP: ; %bb.0:
-; LOOP-NEXT: v_lshlrev_b32_e32 v3, 8, v2
-; LOOP-NEXT: v_and_b32_e32 v4, 0xff, v2
+; LOOP-NEXT: v_and_b32_e32 v3, 0xff, v2
+; LOOP-NEXT: v_lshlrev_b32_e32 v4, 8, v3
+; LOOP-NEXT: v_or_b32_e32 v4, v3, v4
+; LOOP-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; LOOP-NEXT: v_or_b32_e32 v4, v4, v5
+; LOOP-NEXT: v_lshlrev_b32_e32 v3, 24, v3
; LOOP-NEXT: v_or_b32_e32 v3, v4, v3
-; LOOP-NEXT: v_lshlrev_b32_e32 v4, 16, v3
-; LOOP-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; LOOP-NEXT: v_or_b32_e32 v3, v3, v4
-; LOOP-NEXT: v_mov_b32_e32 v4, v3
-; LOOP-NEXT: v_mov_b32_e32 v5, v3
-; LOOP-NEXT: v_mov_b32_e32 v6, v3
-; LOOP-NEXT: v_mov_b32_e32 v7, v3
-; LOOP-NEXT: v_mov_b32_e32 v8, v3
-; LOOP-NEXT: v_mov_b32_e32 v9, v3
-; LOOP-NEXT: v_mov_b32_e32 v10, v3
; LOOP-NEXT: s_mov_b64 s[0:1], 0
-; LOOP-NEXT: s_mov_b32 s3, 0xf000
+; LOOP-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; LOOP-NEXT: v_bfe_u32 v5, v3, 8, 8
; LOOP-NEXT: s_mov_b32 s2, 0
-; LOOP-NEXT: v_lshrrev_b32_e32 v11, 8, v3
-; LOOP-NEXT: v_lshrrev_b32_e32 v12, 24, v4
-; LOOP-NEXT: v_lshrrev_b32_e32 v13, 16, v4
-; LOOP-NEXT: v_lshrrev_b32_e32 v14, 8, v4
-; LOOP-NEXT: v_lshrrev_b32_e32 v15, 24, v5
-; LOOP-NEXT: v_lshrrev_b32_e32 v16, 16, v5
-; LOOP-NEXT: v_lshrrev_b32_e32 v17, 8, v5
-; LOOP-NEXT: v_lshrrev_b32_e32 v18, 24, v6
-; LOOP-NEXT: v_lshrrev_b32_e32 v19, 16, v6
-; LOOP-NEXT: v_lshrrev_b32_e32 v20, 8, v6
-; LOOP-NEXT: v_lshrrev_b32_e32 v21, 24, v7
-; LOOP-NEXT: v_lshrrev_b32_e32 v22, 16, v7
-; LOOP-NEXT: v_lshrrev_b32_e32 v23, 8, v7
-; LOOP-NEXT: v_lshrrev_b32_e32 v24, 24, v8
-; LOOP-NEXT: v_lshrrev_b32_e32 v25, 16, v8
-; LOOP-NEXT: v_lshrrev_b32_e32 v26, 8, v8
-; LOOP-NEXT: v_lshrrev_b32_e32 v27, 24, v9
-; LOOP-NEXT: v_lshrrev_b32_e32 v28, 16, v9
-; LOOP-NEXT: v_lshrrev_b32_e32 v29, 8, v9
+; LOOP-NEXT: s_mov_b32 s3, 0xf000
+; LOOP-NEXT: v_lshrrev_b32_e32 v6, 24, v3
; LOOP-NEXT: .LBB0_1: ; %static-memset-expansion-main-body
; LOOP-NEXT: ; =>This Inner Loop Header: Depth=1
-; LOOP-NEXT: v_lshrrev_b32_e32 v30, 24, v10
-; LOOP-NEXT: buffer_store_byte v30, v[0:1], s[0:3], 0 addr64 offset:31
-; LOOP-NEXT: s_waitcnt expcnt(0)
-; LOOP-NEXT: v_lshrrev_b32_e32 v30, 16, v10
-; LOOP-NEXT: buffer_store_byte v30, v[0:1], s[0:3], 0 addr64 offset:30
-; LOOP-NEXT: s_waitcnt expcnt(0)
-; LOOP-NEXT: v_lshrrev_b32_e32 v30, 24, v3
-; LOOP-NEXT: buffer_store_byte v30, v[0:1], s[0:3], 0 addr64 offset:3
-; LOOP-NEXT: s_waitcnt expcnt(0)
-; LOOP-NEXT: v_lshrrev_b32_e32 v30, 16, v3
-; LOOP-NEXT: buffer_store_byte v30, v[0:1], s[0:3], 0 addr64 offset:2
-; LOOP-NEXT: s_waitcnt expcnt(0)
-; LOOP-NEXT: v_lshrrev_b32_e32 v30, 8, v10
-; LOOP-NEXT: buffer_store_byte v30, v[0:1], s[0:3], 0 addr64 offset:29
-; LOOP-NEXT: buffer_store_byte v11, v[0:1], s[0:3], 0 addr64 offset:1
; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64
-; LOOP-NEXT: buffer_store_byte v12, v[0:1], s[0:3], 0 addr64 offset:7
-; LOOP-NEXT: buffer_store_byte v13, v[0:1], s[0:3], 0 addr64 offset:6
-; LOOP-NEXT: buffer_store_byte v14, v[0:1], s[0:3], 0 addr64 offset:5
-; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:4
-; LOOP-NEXT: buffer_store_byte v15, v[0:1], s[0:3], 0 addr64 offset:11
-; LOOP-NEXT: buffer_store_byte v16, v[0:1], s[0:3], 0 addr64 offset:10
-; LOOP-NEXT: buffer_store_byte v17, v[0:1], s[0:3], 0 addr64 offset:9
-; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:8
-; LOOP-NEXT: buffer_store_byte v18, v[0:1], s[0:3], 0 addr64 offset:15
-; LOOP-NEXT: buffer_store_byte v19, v[0:1], s[0:3], 0 addr64 offset:14
-; LOOP-NEXT: buffer_store_byte v20, v[0:1], s[0:3], 0 addr64 offset:13
-; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:12
-; LOOP-NEXT: buffer_store_byte v21, v[0:1], s[0:3], 0 addr64 offset:19
-; LOOP-NEXT: buffer_store_byte v22, v[0:1], s[0:3], 0 addr64 offset:18
-; LOOP-NEXT: buffer_store_byte v23, v[0:1], s[0:3], 0 addr64 offset:17
-; LOOP-NEXT: buffer_store_byte v7, v[0:1], s[0:3], 0 addr64 offset:16
-; LOOP-NEXT: buffer_store_byte v24, v[0:1], s[0:3], 0 addr64 offset:23
-; LOOP-NEXT: buffer_store_byte v25, v[0:1], s[0:3], 0 addr64 offset:22
-; LOOP-NEXT: buffer_store_byte v26, v[0:1], s[0:3], 0 addr64 offset:21
-; LOOP-NEXT: buffer_store_byte v8, v[0:1], s[0:3], 0 addr64 offset:20
-; LOOP-NEXT: buffer_store_byte v27, v[0:1], s[0:3], 0 addr64 offset:27
-; LOOP-NEXT: buffer_store_byte v28, v[0:1], s[0:3], 0 addr64 offset:26
-; LOOP-NEXT: buffer_store_byte v29, v[0:1], s[0:3], 0 addr64 offset:25
-; LOOP-NEXT: buffer_store_byte v9, v[0:1], s[0:3], 0 addr64 offset:24
-; LOOP-NEXT: buffer_store_byte v10, v[0:1], s[0:3], 0 addr64 offset:28
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:1
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:2
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:3
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:4
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:5
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:6
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:7
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:8
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:9
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:10
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:11
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:12
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:13
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:14
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:15
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:16
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:17
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:18
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:19
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:20
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:21
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:22
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:23
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:24
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:25
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:26
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:27
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:28
+; LOOP-NEXT: buffer_store_byte v5, v[0:1], s[0:3], 0 addr64 offset:29
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:30
+; LOOP-NEXT: buffer_store_byte v6, v[0:1], s[0:3], 0 addr64 offset:31
; LOOP-NEXT: s_add_u32 s0, s0, 32
; LOOP-NEXT: s_addc_u32 s1, s1, 0
; LOOP-NEXT: s_cmp_lt_u32 s0, 32
; LOOP-NEXT: s_cbranch_scc1 .LBB0_1
; LOOP-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; LOOP-NEXT: v_and_b32_e32 v2, 0xff, v2
+; LOOP-NEXT: s_waitcnt expcnt(3)
+; LOOP-NEXT: v_lshlrev_b32_e32 v3, 8, v2
+; LOOP-NEXT: v_or_b32_e32 v3, v2, v3
+; LOOP-NEXT: s_waitcnt expcnt(1)
+; LOOP-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; LOOP-NEXT: v_or_b32_e32 v3, v3, v4
+; LOOP-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; LOOP-NEXT: v_or_b32_e32 v2, v3, v2
+; LOOP-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; LOOP-NEXT: s_mov_b32 s2, 0
; LOOP-NEXT: s_mov_b32 s3, 0xf000
-; LOOP-NEXT: s_mov_b32 s0, s2
-; LOOP-NEXT: s_mov_b32 s1, s2
-; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:35
-; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:34
-; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:33
+; LOOP-NEXT: s_mov_b64 s[0:1], 0
+; LOOP-NEXT: v_bfe_u32 v4, v2, 8, 8
; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:32
+; LOOP-NEXT: buffer_store_byte v4, v[0:1], s[0:3], 0 addr64 offset:33
+; LOOP-NEXT: s_waitcnt expcnt(1)
+; LOOP-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; LOOP-NEXT: buffer_store_byte v3, v[0:1], s[0:3], 0 addr64 offset:34
+; LOOP-NEXT: buffer_store_byte v2, v[0:1], s[0:3], 0 addr64 offset:35
; LOOP-NEXT: s_endpgm
;
; UNROLL-LABEL: memset_p1i8:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
index a7658cea07b1b..a4b0645e0b76e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/lshr.ll
@@ -765,21 +765,22 @@ define <2 x i16> @v_lshr_v2i16(<2 x i16> %value, <2 x i16> %amount) {
; GFX6-LABEL: v_lshr_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, v3, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v3, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_lshr_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, v1, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v1, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_lshr_v2i16:
@@ -801,15 +802,17 @@ define <2 x i16> @v_lshr_v2i16_15(<2 x i16> %value) {
; GFX6-LABEL: v_lshr_v2i16_15:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 15, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0x10001, v0
+; GFX6-NEXT: v_bfe_u32 v1, v0, 15, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 31, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_lshr_v2i16_15:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 15, v0
-; GFX8-NEXT: v_and_b32_e32 v1, 0x10000, v1
+; GFX8-NEXT: v_mov_b32_e32 v1, 31
+; GFX8-NEXT: v_lshrrev_b32_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX8-NEXT: v_lshrrev_b16_e32 v0, 15, v0
; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -832,24 +835,24 @@ define <2 x i16> @v_lshr_v2i16_15(<2 x i16> %value) {
define amdgpu_ps i32 @s_lshr_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amount) {
; GFX6-LABEL: s_lshr_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
; GFX6-NEXT: s_lshr_b32 s3, s1, 16
-; GFX6-NEXT: s_lshr_b32 s0, s0, 16
-; GFX6-NEXT: s_lshr_b32 s0, s0, s3
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_lshr_b32 s1, s2, s1
-; GFX6-NEXT: s_or_b32 s0, s1, s0
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshr_b32 s0, s0, s1
+; GFX6-NEXT: s_lshr_b32 s1, s2, s3
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_lshr_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_lshr_b32 s2, s3, s2
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshr_b32 s0, s0, s1
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_lshr_b32 s1, s2, s3
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_lshr_v2i16:
@@ -879,22 +882,23 @@ define amdgpu_ps i32 @s_lshr_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amou
define amdgpu_ps float @lshr_v2i16_sv(<2 x i16> inreg %value, <2 x i16> %amount) {
; GFX6-LABEL: lshr_v2i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX6-NEXT: s_lshr_b32 s0, s0, 16
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: v_lshr_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: v_lshr_b32_e32 v1, s1, v1
+; GFX6-NEXT: v_lshr_b32_e32 v0, s0, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_lshr_b32_e32 v0, s1, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: lshr_v2i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_lshrrev_b16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b16_e64 v0, v0, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_lshrrev_b16_e64 v1, v0, s0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: lshr_v2i16_sv:
@@ -914,22 +918,23 @@ define amdgpu_ps float @lshr_v2i16_sv(<2 x i16> inreg %value, <2 x i16> %amount)
define amdgpu_ps float @lshr_v2i16_vs(<2 x i16> %value, <2 x i16> inreg %amount) {
; GFX6-LABEL: lshr_v2i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX6-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, s1, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, s0, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, s1, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: lshr_v2i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_lshrrev_b16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, s0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_lshrrev_b16_e32 v1, s0, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: lshr_v2i16_vs:
@@ -961,31 +966,33 @@ define <2 x float> @v_lshr_v4i16(<4 x i16> %value, <4 x i16> %amount) {
; GFX6-LABEL: v_lshr_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, v7, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, v2, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, v5, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, v3, v4
-; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v4
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v3, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v7, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_lshr_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_sdwa v4, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, v2, v0
-; GFX8-NEXT: v_lshrrev_b16_sdwa v2, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, v3, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_lshrrev_b16_e32 v4, v2, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_lshrrev_b16_e32 v2, v3, v1
+; GFX8-NEXT: v_lshrrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_lshr_v4i16:
@@ -1009,38 +1016,38 @@ define <2 x float> @v_lshr_v4i16(<4 x i16> %value, <4 x i16> %amount) {
define amdgpu_ps <2 x i32> @s_lshr_v4i16(<4 x i16> inreg %value, <4 x i16> inreg %amount) {
; GFX6-LABEL: s_lshr_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s4, s0, 0xffff
-; GFX6-NEXT: s_lshr_b32 s5, s2, 16
-; GFX6-NEXT: s_lshr_b32 s0, s0, 16
-; GFX6-NEXT: s_and_b32 s6, s1, 0xffff
+; GFX6-NEXT: s_lshr_b32 s4, s0, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshr_b32 s5, s1, 16
; GFX6-NEXT: s_lshr_b32 s7, s3, 16
-; GFX6-NEXT: s_lshr_b32 s1, s1, 16
-; GFX6-NEXT: s_lshr_b32 s1, s1, s7
-; GFX6-NEXT: s_lshr_b32 s0, s0, s5
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_lshr_b32 s3, s6, s3
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_lshr_b32 s2, s4, s2
-; GFX6-NEXT: s_or_b32 s1, s3, s1
-; GFX6-NEXT: s_or_b32 s0, s2, s0
+; GFX6-NEXT: s_lshr_b32 s0, s0, s2
+; GFX6-NEXT: s_lshr_b32 s2, s4, s6
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshr_b32 s1, s1, s3
+; GFX6-NEXT: s_lshr_b32 s3, s5, s7
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_lshl_b32 s2, s3, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s2
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_lshr_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: s_lshr_b32 s4, s5, s4
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_lshl_b32 s4, s4, 16
-; GFX8-NEXT: s_lshr_b32 s1, s1, s3
-; GFX8-NEXT: s_or_b32 s1, s1, s4
-; GFX8-NEXT: s_lshr_b32 s3, s2, 16
; GFX8-NEXT: s_lshr_b32 s4, s0, 16
-; GFX8-NEXT: s_lshr_b32 s3, s4, s3
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s5, s1, 16
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
; GFX8-NEXT: s_lshr_b32 s0, s0, s2
-; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_lshr_b32 s2, s4, s6
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshr_b32 s1, s1, s3
+; GFX8-NEXT: s_lshr_b32 s3, s5, s7
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_lshl_b32 s2, s3, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_lshr_v4i16:
@@ -1106,51 +1113,55 @@ define <4 x float> @v_lshr_v8i16(<8 x i16> %value, <8 x i16> %amount) {
; GFX6-LABEL: v_lshr_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v14, 0xffff, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v12, 0xffff, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v0, v15, v0
-; GFX6-NEXT: v_and_b32_e32 v10, 0xffff, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v4, v14
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, v13, v1
-; GFX6-NEXT: v_and_b32_e32 v8, 0xffff, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v5, v12
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, v11, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v6, v10
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, v9, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, v7, v8
-; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, v12, v8
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, v5, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, v13, v9
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, v6, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, v14, v10
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, v7, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, v15, v11
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_lshr_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b16_sdwa v8, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v0, v4, v0
-; GFX8-NEXT: v_lshrrev_b16_sdwa v4, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: v_lshrrev_b16_sdwa v4, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_lshrrev_b16_sdwa v4, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshrrev_b16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_lshrrev_b16_e32 v8, v4, v0
+; GFX8-NEXT: v_lshrrev_b16_sdwa v0, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_lshrrev_b16_e32 v4, v5, v1
+; GFX8-NEXT: v_lshrrev_b16_sdwa v1, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_lshrrev_b16_e32 v4, v6, v2
+; GFX8-NEXT: v_lshrrev_b16_sdwa v2, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_lshrrev_b16_e32 v4, v7, v3
+; GFX8-NEXT: v_lshrrev_b16_sdwa v3, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_lshr_v8i16:
@@ -1178,66 +1189,66 @@ define <4 x float> @v_lshr_v8i16(<8 x i16> %value, <8 x i16> %amount) {
define amdgpu_ps <4 x i32> @s_lshr_v8i16(<8 x i16> inreg %value, <8 x i16> inreg %amount) {
; GFX6-LABEL: s_lshr_v8i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s8, s0, 0xffff
-; GFX6-NEXT: s_lshr_b32 s9, s4, 16
-; GFX6-NEXT: s_lshr_b32 s0, s0, 16
-; GFX6-NEXT: s_and_b32 s10, s1, 0xffff
-; GFX6-NEXT: s_lshr_b32 s11, s5, 16
-; GFX6-NEXT: s_lshr_b32 s1, s1, 16
-; GFX6-NEXT: s_and_b32 s12, s2, 0xffff
-; GFX6-NEXT: s_lshr_b32 s13, s6, 16
-; GFX6-NEXT: s_lshr_b32 s2, s2, 16
-; GFX6-NEXT: s_and_b32 s14, s3, 0xffff
+; GFX6-NEXT: s_lshr_b32 s8, s0, 16
+; GFX6-NEXT: s_lshr_b32 s12, s4, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshr_b32 s9, s1, 16
+; GFX6-NEXT: s_lshr_b32 s13, s5, 16
+; GFX6-NEXT: s_lshr_b32 s0, s0, s4
+; GFX6-NEXT: s_lshr_b32 s4, s8, s12
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshr_b32 s10, s2, 16
+; GFX6-NEXT: s_lshr_b32 s14, s6, 16
+; GFX6-NEXT: s_lshr_b32 s1, s1, s5
+; GFX6-NEXT: s_lshr_b32 s5, s9, s13
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_lshr_b32 s11, s3, 16
; GFX6-NEXT: s_lshr_b32 s15, s7, 16
-; GFX6-NEXT: s_lshr_b32 s3, s3, 16
-; GFX6-NEXT: s_lshr_b32 s3, s3, s15
-; GFX6-NEXT: s_lshr_b32 s2, s2, s13
-; GFX6-NEXT: s_lshr_b32 s1, s1, s11
-; GFX6-NEXT: s_lshr_b32 s0, s0, s9
-; GFX6-NEXT: s_lshl_b32 s3, s3, 16
-; GFX6-NEXT: s_lshr_b32 s7, s14, s7
-; GFX6-NEXT: s_lshl_b32 s2, s2, 16
-; GFX6-NEXT: s_lshr_b32 s6, s12, s6
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
-; GFX6-NEXT: s_lshr_b32 s5, s10, s5
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_lshr_b32 s4, s8, s4
-; GFX6-NEXT: s_or_b32 s3, s7, s3
-; GFX6-NEXT: s_or_b32 s2, s6, s2
-; GFX6-NEXT: s_or_b32 s1, s5, s1
-; GFX6-NEXT: s_or_b32 s0, s4, s0
+; GFX6-NEXT: s_lshr_b32 s2, s2, s6
+; GFX6-NEXT: s_lshr_b32 s6, s10, s14
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_lshl_b32 s4, s5, 16
+; GFX6-NEXT: s_lshr_b32 s3, s3, s7
+; GFX6-NEXT: s_lshr_b32 s7, s11, s15
+; GFX6-NEXT: s_or_b32 s1, s1, s4
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_lshl_b32 s4, s7, 16
+; GFX6-NEXT: s_or_b32 s3, s3, s4
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_lshr_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s8, s7, 16
-; GFX8-NEXT: s_lshr_b32 s9, s3, 16
-; GFX8-NEXT: s_lshr_b32 s8, s9, s8
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_lshl_b32 s8, s8, 16
-; GFX8-NEXT: s_lshr_b32 s3, s3, s7
-; GFX8-NEXT: s_or_b32 s3, s3, s8
-; GFX8-NEXT: s_lshr_b32 s7, s6, 16
-; GFX8-NEXT: s_lshr_b32 s8, s2, 16
-; GFX8-NEXT: s_lshr_b32 s7, s8, s7
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_lshl_b32 s7, s7, 16
-; GFX8-NEXT: s_lshr_b32 s2, s2, s6
-; GFX8-NEXT: s_or_b32 s2, s2, s7
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s1, 16
-; GFX8-NEXT: s_lshr_b32 s6, s7, s6
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_lshr_b32 s1, s1, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s6
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_lshr_b32 s6, s0, 16
-; GFX8-NEXT: s_lshr_b32 s5, s6, s5
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_lshl_b32 s5, s5, 16
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
; GFX8-NEXT: s_lshr_b32 s0, s0, s4
-; GFX8-NEXT: s_or_b32 s0, s0, s5
+; GFX8-NEXT: s_lshr_b32 s4, s8, s12
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: s_lshr_b32 s1, s1, s5
+; GFX8-NEXT: s_lshr_b32 s5, s9, s13
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: s_lshr_b32 s2, s2, s6
+; GFX8-NEXT: s_lshr_b32 s6, s10, s14
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_lshl_b32 s4, s5, 16
+; GFX8-NEXT: s_lshr_b32 s3, s3, s7
+; GFX8-NEXT: s_lshr_b32 s7, s11, s15
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_lshl_b32 s4, s6, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_lshl_b32 s4, s7, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_lshr_v8i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
index 3e70134a4a303..394ba5e906f1a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GCN,GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GCN,GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16, -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
@@ -438,27 +438,29 @@ define i32 @v_mul_i32(i32 %num, i32 %den) {
define amdgpu_ps <2 x i16> @s_mul_v2i16(<2 x i16> inreg %num, <2 x i16> inreg %den) {
; GFX7-LABEL: s_mul_v2i16:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_lshr_b32 s2, s1, 16
-; GFX7-NEXT: s_lshr_b32 s3, s0, 16
-; GFX7-NEXT: s_mul_i32 s2, s3, s2
+; GFX7-NEXT: s_lshr_b32 s2, s0, 16
+; GFX7-NEXT: s_lshr_b32 s3, s1, 16
; GFX7-NEXT: s_mul_i32 s0, s0, s1
-; GFX7-NEXT: s_add_i32 s2, s2, s2
+; GFX7-NEXT: s_mul_i32 s1, s2, s3
+; GFX7-NEXT: s_add_i32 s1, s1, s1
; GFX7-NEXT: s_add_i32 s0, s0, s0
-; GFX7-NEXT: s_lshl_b32 s1, s2, 16
-; GFX7-NEXT: s_and_b32 s0, s0, 0xfffe
+; GFX7-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX7-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX7-NEXT: s_lshl_b32 s1, s1, 16
; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_mul_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_mul_i32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_mul_i32 s0, s0, s1
-; GFX8-NEXT: s_add_i32 s2, s2, s2
+; GFX8-NEXT: s_mul_i32 s1, s2, s3
+; GFX8-NEXT: s_add_i32 s1, s1, s1
; GFX8-NEXT: s_add_i32 s0, s0, s0
-; GFX8-NEXT: s_lshl_b32 s1, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
@@ -544,9 +546,9 @@ define <2 x i16> @v_mul_v2i16(<2 x i16> %num, <2 x i16> %den) {
; GFX8-LABEL: v_mul_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_lo_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_mul_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll
index 50929a6dbf0b8..6755a2faf2950 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/mul.v2i16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdhsa -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX10 %s
@@ -14,9 +14,9 @@ define <2 x i16> @v_mul_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX8-LABEL: v_mul_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_lo_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_v2i16:
@@ -39,9 +39,9 @@ define <2 x i16> @v_mul_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_lo_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_v2i16_fneg_lhs:
@@ -66,9 +66,9 @@ define <2 x i16> @v_mul_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
-; GFX8-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_mul_lo_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_mul_v2i16_fneg_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
index eb838751ea24d..4c0436d5f6318 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/no-ctlz-from-umul-to-lshr-in-postlegalizer.ll
@@ -1,38 +1,83 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -O0 -global-isel=true -global-isel-abort=2 %s -o - | FileCheck %s
+; RUN: llc -mtriple=amdgpu9.0a-amd-amdhsa -O0 -global-isel=true %s -o - | FileCheck %s
define void @test(ptr %p) {
; CHECK-LABEL: test:
; CHECK: ; %bb.0:
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v2, v0
-; CHECK-NEXT: ; kill: def $vgpr2 killed $vgpr2 def $vgpr2_vgpr3 killed $exec
-; CHECK-NEXT: v_mov_b32_e32 v3, v1
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
-; CHECK-NEXT: ; implicit-def: $sgpr4
-; CHECK-NEXT: v_mov_b32_e32 v4, s4
-; CHECK-NEXT: flat_store_byte v[0:1], v4 offset:1
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
-; CHECK-NEXT: ; implicit-def: $sgpr4
-; CHECK-NEXT: v_mov_b32_e32 v4, s4
-; CHECK-NEXT: flat_store_byte v[0:1], v4
-; CHECK-NEXT: v_pk_mov_b32 v[0:1], v[2:3], v[2:3] op_sel:[0,1]
-; CHECK-NEXT: ; implicit-def: $sgpr4
-; CHECK-NEXT: v_mov_b32_e32 v4, s4
-; CHECK-NEXT: flat_store_byte v[0:1], v4 offset:2
-; CHECK-NEXT: s_mov_b64 s[6:7], 2
-; CHECK-NEXT: v_mov_b32_e32 v0, v2
-; CHECK-NEXT: s_mov_b32 s4, s6
-; CHECK-NEXT: v_mov_b32_e32 v1, v3
-; CHECK-NEXT: s_mov_b32 s6, s7
-; CHECK-NEXT: v_add_co_u32_e64 v0, s[4:5], v0, s4
-; CHECK-NEXT: v_mov_b32_e32 v2, s6
-; CHECK-NEXT: v_addc_co_u32_e64 v2, s[4:5], v1, v2, s[4:5]
+; CHECK-NEXT: v_mov_b32_e32 v2, v1
; CHECK-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; CHECK-NEXT: v_mov_b32_e32 v1, v2
-; CHECK-NEXT: ; implicit-def: $sgpr4
+; CHECK-NEXT: s_mov_b32 s8, 16
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
+; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
+; CHECK-NEXT: s_mov_b32 s5, 0x4f7ffffe
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
+; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
+; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
+; CHECK-NEXT: v_readfirstlane_b32 s4, v2
+; CHECK-NEXT: s_mov_b32 s7, 0
+; CHECK-NEXT: s_mul_hi_u32 s7, s4, s7
+; CHECK-NEXT: s_add_i32 s4, s4, s7
+; CHECK-NEXT: s_mul_hi_u32 s4, s4, s8
+; CHECK-NEXT: s_mov_b32 s7, 2
+; CHECK-NEXT: s_add_i32 s4, s4, s7
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
+; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
+; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
+; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v2
+; CHECK-NEXT: s_mov_b32 s9, 0
+; CHECK-NEXT: s_mul_hi_u32 s9, s7, s9
+; CHECK-NEXT: s_add_i32 s7, s7, s9
+; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
+; CHECK-NEXT: s_mov_b32 s9, 2
+; CHECK-NEXT: s_add_i32 s9, s7, s9
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
+; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
+; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
+; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
+; CHECK-NEXT: v_readfirstlane_b32 s7, v2
+; CHECK-NEXT: s_mov_b32 s10, 0
+; CHECK-NEXT: s_mul_hi_u32 s10, s7, s10
+; CHECK-NEXT: s_add_i32 s7, s7, s10
+; CHECK-NEXT: s_mul_hi_u32 s7, s7, s8
+; CHECK-NEXT: s_mov_b32 s10, 2
+; CHECK-NEXT: s_add_i32 s7, s7, s10
+; CHECK-NEXT: v_mov_b32_e32 v2, s6
+; CHECK-NEXT: v_cvt_f32_ubyte0_e64 v2, v2
+; CHECK-NEXT: v_rcp_iflag_f32_e64 v2, v2
+; CHECK-NEXT: v_mov_b32_e32 v3, s5
+; CHECK-NEXT: v_mul_f32_e64 v2, v2, v3
+; CHECK-NEXT: v_cvt_u32_f32_e64 v2, v2
+; CHECK-NEXT: v_readfirstlane_b32 s5, v2
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mul_hi_u32 s6, s5, s6
+; CHECK-NEXT: s_add_i32 s5, s5, s6
+; CHECK-NEXT: s_mul_hi_u32 s5, s5, s8
+; CHECK-NEXT: s_mov_b32 s6, 2
+; CHECK-NEXT: s_add_i32 s5, s5, s6
+; CHECK-NEXT: s_mov_b32 s6, 0xff
+; CHECK-NEXT: s_and_b32 s4, s4, s6
+; CHECK-NEXT: s_and_b32 s9, s9, s6
+; CHECK-NEXT: s_mov_b32 s10, 8
+; CHECK-NEXT: s_lshl_b32 s9, s9, s10
+; CHECK-NEXT: s_or_b32 s4, s4, s9
+; CHECK-NEXT: s_and_b32 s7, s7, s6
+; CHECK-NEXT: s_lshl_b32 s7, s7, s8
+; CHECK-NEXT: s_or_b32 s4, s4, s7
+; CHECK-NEXT: s_and_b32 s5, s5, s6
+; CHECK-NEXT: s_mov_b32 s6, 24
+; CHECK-NEXT: s_lshl_b32 s5, s5, s6
+; CHECK-NEXT: s_or_b32 s4, s4, s5
; CHECK-NEXT: v_mov_b32_e32 v2, s4
-; CHECK-NEXT: flat_store_byte v[0:1], v2 offset:1
+; CHECK-NEXT: flat_store_dword v[0:1], v2
; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
%B = udiv <4 x i8> splat (i8 16), zeroinitializer
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
index 96306684854bf..e8aecb318b07b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/orn2.ll
@@ -1,9 +1,9 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; FIXME: regbankcombiner regression, related to:
; - looking through copy and splitting G_CONSTANT i64 to two i32 constants
@@ -699,31 +699,49 @@ define <2 x i16> @v_orn2_v2i16(<2 x i16> %src0, <2 x i16> %src1) {
define amdgpu_ps i48 @s_orn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX6-LABEL: s_orn2_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_orn2_b32 s0, s3, s5
-; GFX6-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX6-NEXT: s_orn2_b32 s0, s2, s4
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_mov_b32 s0, -1
+; GFX6-NEXT: s_mov_b32 s1, 0xffff
+; GFX6-NEXT: s_or_b32 s4, s4, s7
+; GFX6-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX6-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_orn2_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s4, 16
-; GFX9-NEXT: s_lshr_b32 s5, s2, 16
-; GFX9-NEXT: s_orn2_b32 s0, s5, s0
-; GFX9-NEXT: s_orn2_b32 s2, s2, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
-; GFX9-NEXT: s_or_b32 s1, s3, s1
+; GFX9-NEXT: s_mov_b64 s[0:1], -1
+; GFX9-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_orn2_v3i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s0, s4, 16
-; GFX10-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10-NEXT: s_orn2_b32 s0, s1, s0
-; GFX10-NEXT: s_orn2_b32 s1, s2, s4
-; GFX10-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX10-NEXT: s_or_b32 s1, s3, s2
+; GFX10-NEXT: s_mov_b64 s[0:1], -1
+; GFX10-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_orn2_v3i16:
@@ -737,13 +755,14 @@ define amdgpu_ps i48 @s_orn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1)
;
; GFX11-FAKE16-LABEL: s_orn2_v3i16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s2, s4
-; GFX11-FAKE16-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s3, s2
+; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-FAKE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-FAKE16-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%or = or <3 x i16> %src0, %not.src1
@@ -754,31 +773,49 @@ define amdgpu_ps i48 @s_orn2_v3i16(<3 x i16> inreg %src0, <3 x i16> inreg %src1)
define amdgpu_ps i48 @s_orn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX6-LABEL: s_orn2_v3i16_commute:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_orn2_b32 s0, s3, s5
-; GFX6-NEXT: s_and_b32 s1, s0, 0xffff
-; GFX6-NEXT: s_orn2_b32 s0, s2, s4
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_mov_b32 s0, -1
+; GFX6-NEXT: s_mov_b32 s1, 0xffff
+; GFX6-NEXT: s_or_b32 s4, s4, s7
+; GFX6-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX6-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_orn2_v3i16_commute:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_xor_b32 s1, s5, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s4, 16
-; GFX9-NEXT: s_lshr_b32 s5, s2, 16
-; GFX9-NEXT: s_orn2_b32 s0, s5, s0
-; GFX9-NEXT: s_orn2_b32 s2, s2, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
-; GFX9-NEXT: s_or_b32 s1, s1, s3
+; GFX9-NEXT: s_mov_b64 s[0:1], -1
+; GFX9-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX9-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_orn2_v3i16_commute:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s0, s4, 16
-; GFX10-NEXT: s_lshr_b32 s1, s2, 16
-; GFX10-NEXT: s_orn2_b32 s0, s1, s0
-; GFX10-NEXT: s_orn2_b32 s1, s2, s4
-; GFX10-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX10-NEXT: s_or_b32 s1, s2, s3
+; GFX10-NEXT: s_mov_b64 s[0:1], -1
+; GFX10-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX10-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_orn2_v3i16_commute:
@@ -792,13 +829,14 @@ define amdgpu_ps i48 @s_orn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inre
;
; GFX11-FAKE16-LABEL: s_orn2_v3i16_commute:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s2, s4
-; GFX11-FAKE16-NEXT: s_xor_b32 s2, s5, 0xffff
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_or_b32 s1, s2, s3
+; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-FAKE16-NEXT: s_xor_b64 s[0:1], s[4:5], s[0:1]
+; GFX11-FAKE16-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%or = or <3 x i16> %not.src1, %src0
@@ -809,54 +847,64 @@ define amdgpu_ps i48 @s_orn2_v3i16_commute(<3 x i16> inreg %src0, <3 x i16> inre
define amdgpu_ps { i48, i48 } @s_orn2_v3i16_multi_use(<3 x i16> inreg %src0, <3 x i16> inreg %src1) {
; GFX6-LABEL: s_orn2_v3i16_multi_use:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s0, s2, 16
-; GFX6-NEXT: s_lshr_b32 s1, s4, 16
-; GFX6-NEXT: s_not_b32 s7, s1
-; GFX6-NEXT: s_orn2_b32 s0, s0, s1
-; GFX6-NEXT: s_orn2_b32 s1, s2, s4
-; GFX6-NEXT: s_andn2_b32 s6, 0xffff, s4
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
; GFX6-NEXT: s_lshl_b32 s7, s7, 16
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_or_b32 s6, s6, s7
-; GFX6-NEXT: s_andn2_b32 s7, 0xffff, s5
-; GFX6-NEXT: s_or_b32 s0, s1, s0
-; GFX6-NEXT: s_orn2_b32 s1, s3, s5
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_mov_b32 s0, -1
+; GFX6-NEXT: s_mov_b32 s1, 0xffff
+; GFX6-NEXT: s_or_b32 s4, s4, s7
+; GFX6-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX6-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX6-NEXT: s_and_b32 s0, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s6, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s3, 0xffff
+; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[4:5]
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s7, 16
; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_mov_b32 s2, s6
-; GFX6-NEXT: s_mov_b32 s3, s7
+; GFX6-NEXT: s_or_b32 s2, s2, s3
+; GFX6-NEXT: s_and_b32 s3, s5, 0xffff
; GFX6-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_orn2_v3i16_multi_use:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s0, s4, 16
-; GFX9-NEXT: s_not_b32 s1, s0
-; GFX9-NEXT: s_not_b32 s6, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX9-NEXT: s_lshr_b32 s1, s2, 16
-; GFX9-NEXT: s_not_b32 s7, s5
-; GFX9-NEXT: s_orn2_b32 s0, s1, s0
-; GFX9-NEXT: s_orn2_b32 s1, s2, s4
-; GFX9-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX9-NEXT: s_orn2_b32 s1, s3, s5
-; GFX9-NEXT: s_mov_b32 s2, s6
-; GFX9-NEXT: s_mov_b32 s3, s7
+; GFX9-NEXT: s_mov_b64 s[0:1], -1
+; GFX9-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX9-NEXT: s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s6, s4, 16
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX9-NEXT: s_lshl_b32 s3, s6, 16
+; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX9-NEXT: s_or_b32 s2, s2, s3
+; GFX9-NEXT: s_and_b32 s3, s5, 0xffff
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_orn2_v3i16_multi_use:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s0, s4, 16
-; GFX10-NEXT: s_not_b32 s6, s4
-; GFX10-NEXT: s_not_b32 s1, s0
-; GFX10-NEXT: s_lshr_b32 s7, s2, 16
-; GFX10-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX10-NEXT: s_orn2_b32 s0, s7, s0
-; GFX10-NEXT: s_orn2_b32 s1, s2, s4
-; GFX10-NEXT: s_not_b32 s4, s5
-; GFX10-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX10-NEXT: s_orn2_b32 s1, s3, s5
-; GFX10-NEXT: s_mov_b32 s2, s6
-; GFX10-NEXT: s_mov_b32 s3, s4
+; GFX10-NEXT: s_mov_b64 s[0:1], -1
+; GFX10-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX10-NEXT: s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX10-NEXT: s_lshr_b32 s3, s4, 16
+; GFX10-NEXT: s_lshr_b32 s2, s0, 16
+; GFX10-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 16
+; GFX10-NEXT: s_lshl_b32 s3, s3, 16
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX10-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX10-NEXT: s_or_b32 s2, s2, s3
+; GFX10-NEXT: s_and_b32 s3, s5, 0xffff
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_orn2_v3i16_multi_use:
@@ -873,18 +921,19 @@ define amdgpu_ps { i48, i48 } @s_orn2_v3i16_multi_use(<3 x i16> inreg %src0, <3
;
; GFX11-FAKE16-LABEL: s_orn2_v3i16_multi_use:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s4, 16
-; GFX11-FAKE16-NEXT: s_not_b32 s6, s4
-; GFX11-FAKE16-NEXT: s_not_b32 s1, s0
-; GFX11-FAKE16-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s6, s6, s1
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s0, s7, s0
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s2, s4
-; GFX11-FAKE16-NEXT: s_not_b32 s4, s5
-; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s1, s0
-; GFX11-FAKE16-NEXT: s_or_not1_b32 s1, s3, s5
-; GFX11-FAKE16-NEXT: s_mov_b32 s2, s6
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, s4
+; GFX11-FAKE16-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-FAKE16-NEXT: s_xor_b64 s[4:5], s[4:5], s[0:1]
+; GFX11-FAKE16-NEXT: s_or_b64 s[0:1], s[2:3], s[4:5]
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s4, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s4, 0xffff
+; GFX11-FAKE16-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_or_b32 s2, s2, s3
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s5, 0xffff
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%not.src1 = xor <3 x i16> %src1, <i16 -1, i16 -1, i16 -1>
%or = or <3 x i16> %src0, %not.src1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index 028f538a501f3..72d1e056bade2 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
define i7 @v_saddsat_i7(i7 %lhs, i7 %rhs) {
; GFX6-LABEL: v_saddsat_i7:
@@ -588,147 +588,249 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX6-LABEL: v_saddsat_v4i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 24, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, 24, v0
-; GFX6-NEXT: v_bfe_i32 v4, v1, 0, 8
-; GFX6-NEXT: v_bfe_i32 v5, v0, 0, 8
-; GFX6-NEXT: v_bfe_i32 v6, v1, 8, 8
-; GFX6-NEXT: v_bfe_i32 v7, v0, 8, 8
-; GFX6-NEXT: v_bfe_i32 v1, v1, 16, 8
-; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 8
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT: s_movk_i32 s4, 0xff80
-; GFX6-NEXT: v_mov_b32_e32 v3, 0x7f
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT: v_min_i32_e32 v10, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v11, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_max_i32_e32 v8, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, v11, v10
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 0x7fffffff, v8
+; GFX6-NEXT: v_max_i32_e32 v1, v10, v1
+; GFX6-NEXT: v_min_i32_e32 v1, v1, v8
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v3
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT: v_min_i32_e32 v8, 0, v1
+; GFX6-NEXT: v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT: v_max_i32_e32 v5, 0, v1
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v11, v8
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT: v_max_i32_e32 v2, v8, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v5
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT: v_min_i32_e32 v6, 0, v2
+; GFX6-NEXT: v_max_i32_e32 v5, 0, v2
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v11, v6
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT: v_max_i32_e32 v3, v6, v3
+; GFX6-NEXT: v_min_i32_e32 v3, v3, v5
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT: v_min_i32_e32 v6, 0, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 24, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT: v_max_i32_e32 v5, 0, v3
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v11, v6
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT: v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 24, v2
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v5
; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, v7, v6
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v4
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v3
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v3
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v4i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 8, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 8, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v6, 8, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v7, 8, v5
-; GFX8-NEXT: v_add_u16_e32 v6, v7, v6
-; GFX8-NEXT: v_add_u16_sdwa v4, sext(v5), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_add_u16_e32 v2, v3, v2
-; GFX8-NEXT: v_add_u16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_e32 v6, 0x7f, v6
-; GFX8-NEXT: v_mov_b32_e32 v7, 0xffffff80
-; GFX8-NEXT: v_min_i16_e32 v4, 0x7f, v4
-; GFX8-NEXT: v_min_i16_e32 v2, 0x7f, v2
-; GFX8-NEXT: v_min_i16_e32 v0, 0x7f, v0
-; GFX8-NEXT: v_max_i16_sdwa v6, v6, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v4, 0xff80, v4
-; GFX8-NEXT: v_max_i16_sdwa v2, v2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v0, 0xff80, v0
-; GFX8-NEXT: v_or_b32_sdwa v4, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, 8
+; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT: v_min_i16_e32 v10, 0, v0
+; GFX8-NEXT: v_lshrrev_b32_sdwa v6, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT: v_max_i16_e32 v9, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v10, 0x8000, v10
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x7fff, v9
+; GFX8-NEXT: v_max_i16_e32 v1, v10, v1
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v9
+; GFX8-NEXT: v_min_i16_e32 v9, 0, v3
+; GFX8-NEXT: v_add_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_max_i16_e32 v1, 0, v3
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_sub_u16_e32 v1, 0x7fff, v1
+; GFX8-NEXT: v_max_i16_e32 v6, v9, v6
+; GFX8-NEXT: v_min_i16_e32 v1, v6, v1
+; GFX8-NEXT: v_add_u16_e32 v1, v3, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT: v_min_i16_e32 v7, 0, v3
+; GFX8-NEXT: v_max_i16_e32 v6, 0, v3
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_sub_u16_e32 v6, 0x7fff, v6
+; GFX8-NEXT: v_max_i16_e32 v4, v7, v4
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v6
+; GFX8-NEXT: v_add_u16_e32 v3, v3, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v5
+; GFX8-NEXT: v_min_i16_e32 v7, 0, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v8
+; GFX8-NEXT: v_max_i16_e32 v6, 0, v4
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_ashrrev_i16_e32 v1, 8, v1
+; GFX8-NEXT: v_sub_u16_e32 v6, 0x7fff, v6
+; GFX8-NEXT: v_max_i16_e32 v5, v7, v5
+; GFX8-NEXT: v_ashrrev_i16_e32 v0, 8, v0
+; GFX8-NEXT: v_ashrrev_i16_e32 v3, 8, v3
+; GFX8-NEXT: v_min_i16_e32 v5, v5, v6
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_add_u16_e32 v4, v4, v5
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-NEXT: v_ashrrev_i16_e32 v4, 8, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_add_i16 v2, v3, v2 clamp
-; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v0
-; GFX9-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-NEXT: v_add_i16 v3, v4, v3 clamp
-; GFX9-NEXT: s_mov_b32 s5, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v2, v3, v2, s5
-; GFX9-NEXT: v_and_b32_sdwa v3, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_add_i16 v3, v4, v3 clamp
-; GFX9-NEXT: v_mov_b32_e32 v4, 8
-; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_add_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_perm_b32 v0, v0, v3, s5
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
+; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
+; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 24
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_saddsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff00, v1
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffffff00, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff00, v3
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffffff00, v2
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_add_nc_i16 v4, v5, v4 clamp
-; GFX10-NEXT: v_add_nc_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_add_nc_i16 v1, v7, v6 clamp
-; GFX10-NEXT: v_add_nc_i16 v2, v2, v3 clamp
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0xc0c0105
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
+; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-NEXT: v_pk_ashrrev_i16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_saddsat_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff00, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff00, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff00, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff00, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v2.l, v2.h, v2.l clamp
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v0.l, v0.l, v1.l clamp
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v1.l, v3.h, v3.l clamp
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v3.l, v0.h, v1.h clamp
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v6 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_i16 v1, v2, v1 clamp
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_pk_ashrrev_i16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_saddsat_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffffff00, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffffff00, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffffff00, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffffff00, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v4, v5, v4 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v0, v0, v1 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v1, v7, v6 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v2, v2, v3 clamp
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v4, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v3, 16, v5
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v6, v0, 16
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_i16 v2, v2, v3 clamp
+; GFX11-FAKE16-NEXT: v_pk_add_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xff, v1, v2
+; GFX11-FAKE16-NEXT: v_or3_b32 v0, v1, v3, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -740,171 +842,366 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
define amdgpu_ps i32 @s_saddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
; GFX6-LABEL: s_saddsat_v4i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s2, s1, 24
-; GFX6-NEXT: s_ashr_i32 s3, s0, 24
-; GFX6-NEXT: s_sext_i32_i8 s4, s1
-; GFX6-NEXT: s_sext_i32_i8 s5, s0
-; GFX6-NEXT: s_bfe_i32 s6, s1, 0x80008
-; GFX6-NEXT: s_bfe_i32 s7, s0, 0x80008
-; GFX6-NEXT: s_bfe_i32 s1, s1, 0x80010
-; GFX6-NEXT: s_bfe_i32 s0, s0, 0x80010
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX6-NEXT: s_lshr_b32 s2, s0, 8
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_lshr_b32 s4, s0, 24
+; GFX6-NEXT: s_lshl_b32 s0, s0, 24
+; GFX6-NEXT: s_min_i32 s9, s0, 0
+; GFX6-NEXT: s_lshr_b32 s5, s1, 8
+; GFX6-NEXT: s_lshr_b32 s6, s1, 16
+; GFX6-NEXT: s_lshr_b32 s7, s1, 24
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_max_i32 s8, s0, 0
+; GFX6-NEXT: s_sub_i32 s9, 0x80000000, s9
+; GFX6-NEXT: s_sub_i32 s8, 0x7fffffff, s8
+; GFX6-NEXT: s_max_i32 s1, s9, s1
+; GFX6-NEXT: s_min_i32 s1, s1, s8
; GFX6-NEXT: s_add_i32 s0, s0, s1
-; GFX6-NEXT: s_add_i32 s3, s3, s2
-; GFX6-NEXT: v_med3_i32 v3, s0, v0, v1
-; GFX6-NEXT: v_med3_i32 v2, s3, v0, v1
-; GFX6-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: s_add_i32 s7, s7, s6
-; GFX6-NEXT: s_add_i32 s5, s5, s4
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX6-NEXT: v_med3_i32 v3, s7, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s5, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 24
+; GFX6-NEXT: s_min_i32 s8, s1, 0
+; GFX6-NEXT: s_lshl_b32 s2, s5, 24
+; GFX6-NEXT: s_max_i32 s5, s1, 0
+; GFX6-NEXT: s_sub_i32 s8, 0x80000000, s8
+; GFX6-NEXT: s_sub_i32 s5, 0x7fffffff, s5
+; GFX6-NEXT: s_max_i32 s2, s8, s2
+; GFX6-NEXT: s_min_i32 s2, s2, s5
+; GFX6-NEXT: s_add_i32 s1, s1, s2
+; GFX6-NEXT: s_lshl_b32 s2, s3, 24
+; GFX6-NEXT: s_lshl_b32 s3, s6, 24
+; GFX6-NEXT: s_min_i32 s6, s2, 0
+; GFX6-NEXT: s_max_i32 s5, s2, 0
+; GFX6-NEXT: s_sub_i32 s6, 0x80000000, s6
+; GFX6-NEXT: s_sub_i32 s5, 0x7fffffff, s5
+; GFX6-NEXT: s_max_i32 s3, s6, s3
+; GFX6-NEXT: s_min_i32 s3, s3, s5
+; GFX6-NEXT: s_add_i32 s2, s2, s3
+; GFX6-NEXT: s_lshl_b32 s3, s4, 24
+; GFX6-NEXT: s_min_i32 s6, s3, 0
+; GFX6-NEXT: s_ashr_i32 s1, s1, 24
+; GFX6-NEXT: s_lshl_b32 s4, s7, 24
+; GFX6-NEXT: s_max_i32 s5, s3, 0
+; GFX6-NEXT: s_sub_i32 s6, 0x80000000, s6
+; GFX6-NEXT: s_ashr_i32 s0, s0, 24
+; GFX6-NEXT: s_sub_i32 s5, 0x7fffffff, s5
+; GFX6-NEXT: s_max_i32 s4, s6, s4
+; GFX6-NEXT: s_and_b32 s1, s1, 0xff
+; GFX6-NEXT: s_ashr_i32 s2, s2, 24
+; GFX6-NEXT: s_min_i32 s4, s4, s5
+; GFX6-NEXT: s_and_b32 s0, s0, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-NEXT: s_add_i32 s3, s3, s4
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s2, 0xff
+; GFX6-NEXT: s_ashr_i32 s3, s3, 24
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s3, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_saddsat_v4i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s0, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: s_ashr_i32 s6, s1, 24
-; GFX8-NEXT: s_ashr_i32 s7, s0, 24
-; GFX8-NEXT: s_sext_i32_i16 s2, s1
-; GFX8-NEXT: s_sext_i32_i16 s3, s0
-; GFX8-NEXT: s_bfe_i32 s4, s4, 0x80000
-; GFX8-NEXT: s_bfe_i32 s5, s5, 0x80000
-; GFX8-NEXT: s_add_i32 s7, s7, s6
-; GFX8-NEXT: s_ashr_i32 s2, s2, 8
-; GFX8-NEXT: s_ashr_i32 s3, s3, 8
-; GFX8-NEXT: s_sext_i32_i16 s6, s7
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x80000
-; GFX8-NEXT: v_med3_i32 v2, s6, v0, v1
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_add_i32 s3, s3, s2
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: v_med3_i32 v3, s4, v0, v1
-; GFX8-NEXT: s_sext_i32_i16 s2, s3
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: s_lshr_b32 s4, s0, 24
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s8, s0
+; GFX8-NEXT: s_max_i32 s9, s8, 0
+; GFX8-NEXT: s_min_i32 s8, s8, 0
+; GFX8-NEXT: s_lshr_b32 s5, s1, 8
+; GFX8-NEXT: s_lshr_b32 s6, s1, 16
+; GFX8-NEXT: s_lshr_b32 s7, s1, 24
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_sub_i32 s8, 0x8000, s8
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sub_i32 s9, 0x7fff, s9
+; GFX8-NEXT: s_max_i32 s1, s8, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s8, s9
+; GFX8-NEXT: s_min_i32 s1, s1, s8
; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_med3_i32 v3, s2, v0, v1
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s5, 8
+; GFX8-NEXT: s_sext_i32_i16 s5, s1
+; GFX8-NEXT: s_max_i32 s8, s5, 0
+; GFX8-NEXT: s_min_i32 s5, s5, 0
+; GFX8-NEXT: s_sub_i32 s5, 0x8000, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sub_i32 s8, 0x7fff, s8
+; GFX8-NEXT: s_max_i32 s2, s5, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s5, s8
+; GFX8-NEXT: s_min_i32 s2, s2, s5
+; GFX8-NEXT: s_add_i32 s1, s1, s2
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: s_sext_i32_i16 s5, s2
+; GFX8-NEXT: s_lshl_b32 s3, s6, 8
+; GFX8-NEXT: s_max_i32 s6, s5, 0
+; GFX8-NEXT: s_min_i32 s5, s5, 0
+; GFX8-NEXT: s_sub_i32 s5, 0x8000, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sub_i32 s6, 0x7fff, s6
+; GFX8-NEXT: s_max_i32 s3, s5, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s5, s6
+; GFX8-NEXT: s_min_i32 s3, s3, s5
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: s_lshl_b32 s3, s4, 8
+; GFX8-NEXT: s_sext_i32_i16 s5, s3
+; GFX8-NEXT: s_max_i32 s6, s5, 0
+; GFX8-NEXT: s_min_i32 s5, s5, 0
+; GFX8-NEXT: s_lshl_b32 s4, s7, 8
+; GFX8-NEXT: s_sub_i32 s5, 0x8000, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sub_i32 s6, 0x7fff, s6
+; GFX8-NEXT: s_max_i32 s4, s5, s4
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s1, s1, 8
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s6
+; GFX8-NEXT: s_ashr_i32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_ashr_i32 s2, s2, 8
+; GFX8-NEXT: s_add_i32 s3, s3, s4
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xff
+; GFX8-NEXT: s_ashr_i32 s3, s3, 8
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s3, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 24
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_v4i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s3, s1, 16
-; GFX9-NEXT: s_lshr_b32 s5, s1, 24
-; GFX9-NEXT: s_and_b32 s6, s1, 0xff00
-; GFX9-NEXT: s_lshl_b32 s1, s1, 8
-; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
; GFX9-NEXT: s_lshr_b32 s4, s0, 24
-; GFX9-NEXT: s_and_b32 s7, s0, 0xff00
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: s_lshl_b32 s0, s0, 8
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_lshl_b32 s5, s5, 8
-; GFX9-NEXT: v_add_i16 v0, s7, v0 clamp
-; GFX9-NEXT: v_add_i16 v1, s0, v1 clamp
-; GFX9-NEXT: s_mov_b32 s0, 0xc0c0105
-; GFX9-NEXT: s_lshl_b32 s1, s3, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
; GFX9-NEXT: s_lshl_b32 s4, s4, 8
-; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_lshl_b32 s2, s2, 8
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_add_i16 v1, s4, v1 clamp
-; GFX9-NEXT: v_add_i16 v2, s2, v2 clamp
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s0
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: s_lshr_b32 s5, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshr_b32 s6, s1, 16
+; GFX9-NEXT: s_lshr_b32 s7, s1, 24
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s6, s7
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT: v_pk_add_i16 v0, s0, v0 clamp
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_pk_add_i16 v0, s2, v0 clamp
+; GFX9-NEXT: s_sext_i32_i16 s2, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_sext_i32_i16 s2, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s1, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT: s_and_b32 s2, s0, 0xff
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s1, s1, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_saddsat_v4i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s2, s0, 16
-; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10-NEXT: s_lshr_b32 s5, s1, 8
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
; GFX10-NEXT: s_lshr_b32 s4, s0, 24
-; GFX10-NEXT: s_lshr_b32 s5, s1, 24
-; GFX10-NEXT: s_lshl_b32 s4, s4, 8
+; GFX10-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10-NEXT: s_lshr_b32 s7, s1, 24
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s4, s6, s7
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x80008
; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s3, s2, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_lshr_b32 s5, s4, 16
+; GFX10-NEXT: s_lshl_b32 s2, s2, 0x80008
; GFX10-NEXT: s_lshl_b32 s3, s3, 8
-; GFX10-NEXT: s_lshl_b32 s2, s2, 8
-; GFX10-NEXT: v_add_nc_i16 v0, s4, s5 clamp
-; GFX10-NEXT: v_add_nc_i16 v1, s2, s3 clamp
-; GFX10-NEXT: s_and_b32 s6, s1, 0xff00
-; GFX10-NEXT: s_and_b32 s7, s0, 0xff00
-; GFX10-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10-NEXT: s_lshl_b32 s0, s0, 8
-; GFX10-NEXT: v_add_nc_i16 v2, s7, s6 clamp
-; GFX10-NEXT: v_add_nc_i16 v3, s0, s1 clamp
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0xc0c0105
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT: s_lshl_b32 s4, s4, 0x80008
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: v_pk_add_i16 v0, s0, s1 clamp
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s4, s5
+; GFX10-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX10-NEXT: v_pk_add_i16 v1, s0, s1 clamp
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_sext_i32_i16 s2, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10-NEXT: s_ashr_i32 s2, s2, s3
+; GFX10-NEXT: s_ashr_i32 s0, s0, 8
+; GFX10-NEXT: s_sext_i32_i16 s4, s1
+; GFX10-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10-NEXT: s_ashr_i32 s3, s4, s3
+; GFX10-NEXT: s_ashr_i32 s1, s1, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_and_b32 s3, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, 16
+; GFX10-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_saddsat_v4i8:
; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: v_pk_add_i16 v0, s2, s3 clamp
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX11-TRUE16-NEXT: v_pk_add_i16 v1, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s2, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_ashr_i32 s2, s2, s3
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s4, s1
+; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s3, s4, s3
+; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s1, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, 0xff00
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s3, 0xff00
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v0.l, s7, s6 clamp
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v1.l, s2, s3 clamp
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s1, 0xff00
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s0, 0xff00
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 8
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v2.l, s5, s4 clamp
-; GFX11-TRUE16-NEXT: v_add_nc_i16 v3.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 24
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_saddsat_v4i8:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-FAKE16-NEXT: s_and_b32 s7, s2, 0xff00
-; GFX11-FAKE16-NEXT: s_and_b32 s6, s3, 0xff00
-; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v0, s7, s6 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v1, s2, s3 clamp
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s1, 0xff00
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s0, 0xff00
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 8
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v2, s5, s4 clamp
-; GFX11-FAKE16-NEXT: v_add_nc_i16 v3, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v2, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: v_pk_add_i16 v0, s2, s3 clamp
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX11-FAKE16-NEXT: v_pk_add_i16 v1, s0, s1 clamp
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s2, s0
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_ashr_i32 s2, s2, s3
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s4, s1
+; GFX11-FAKE16-NEXT: s_ashr_i32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_ashr_i32 s3, s4, s3
+; GFX11-FAKE16-NEXT: s_ashr_i32 s1, s1, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s3, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 24
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -2706,42 +3003,55 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: v_saddsat_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v0
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v6, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v7, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_max_i32_e32 v4, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v7, v6
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, 0x7fffffff, v4
+; GFX6-NEXT: v_max_i32_e32 v1, v6, v1
+; GFX6-NEXT: v_min_i32_e32 v1, v1, v4
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v3
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v4, 0, v1
+; GFX6-NEXT: v_bfrev_b32_e32 v5, -2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_max_i32_e32 v3, 0, v1
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, 0x80000000, v4
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v5, v3
+; GFX6-NEXT: v_max_i32_e32 v2, v4, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v3
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT: v_add_u16_e32 v4, v3, v2
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v4, v3
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v1
-; GFX8-NEXT: v_add_u16_e32 v1, v0, v1
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v1, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v3, 0, v0
+; GFX8-NEXT: v_max_i16_e32 v2, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT: v_sub_u16_e32 v2, 0x7fff, v2
+; GFX8-NEXT: v_max_i16_e32 v3, v3, v1
+; GFX8-NEXT: v_min_i16_e32 v2, v3, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, 0
+; GFX8-NEXT: v_max_i16_sdwa v4, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT: v_sub_u16_e32 v4, 0x7fff, v4
+; GFX8-NEXT: v_max_i16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v4
+; GFX8-NEXT: v_add_u16_e32 v2, v0, v2
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v2i16:
@@ -2762,54 +3072,66 @@ define <2 x i16> @v_saddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: s_saddsat_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s2, s1, 16
-; GFX6-NEXT: s_ashr_i32 s3, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: s_add_i32 s3, s3, s2
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s5, s0, 0
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_max_i32 s4, s0, 0
+; GFX6-NEXT: s_sub_i32 s5, 0x80000000, s5
+; GFX6-NEXT: s_sub_i32 s4, 0x7fffffff, s4
+; GFX6-NEXT: s_max_i32 s1, s5, s1
+; GFX6-NEXT: s_min_i32 s1, s1, s4
; GFX6-NEXT: s_add_i32 s0, s0, s1
-; GFX6-NEXT: v_med3_i32 v2, s3, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: s_min_i32 s4, s1, 0
+; GFX6-NEXT: s_lshl_b32 s2, s3, 16
+; GFX6-NEXT: s_max_i32 s3, s1, 0
+; GFX6-NEXT: s_sub_i32 s4, 0x80000000, s4
+; GFX6-NEXT: s_sub_i32 s3, 0x7fffffff, s3
+; GFX6-NEXT: s_max_i32 s2, s4, s2
+; GFX6-NEXT: s_min_i32 s2, s2, s3
+; GFX6-NEXT: s_add_i32 s1, s1, s2
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_saddsat_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_add_i32 s6, s3, s2
-; GFX8-NEXT: s_sext_i32_i16 s7, s6
-; GFX8-NEXT: s_ashr_i32 s2, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s7, s2
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s1, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
-; GFX8-NEXT: s_lshr_b32 s4, s7, 15
-; GFX8-NEXT: s_xor_b32 s4, s4, 0x8000
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX8-NEXT: s_cselect_b32 s2, s4, s6
-; GFX8-NEXT: s_lshl_b32 s4, s2, 16
-; GFX8-NEXT: s_add_i32 s5, s0, s1
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_sext_i32_i16 s6, s5
-; GFX8-NEXT: s_sext_i32_i16 s2, s0
-; GFX8-NEXT: s_cmp_lt_i32 s1, 0
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, s2
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: s_lshr_b32 s2, s6, 15
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX8-NEXT: s_cselect_b32 s0, s2, s5
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s0
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s1, s4, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s4, s5
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_min_i32 s1, s1, s4
+; GFX8-NEXT: s_add_i32 s0, s0, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s2
+; GFX8-NEXT: s_max_i32 s4, s1, 0
+; GFX8-NEXT: s_min_i32 s1, s1, 0
+; GFX8-NEXT: s_sub_i32 s1, 0x8000, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sub_i32 s4, 0x7fff, s4
+; GFX8-NEXT: s_max_i32 s1, s1, s3
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s3, s4
+; GFX8-NEXT: s_min_i32 s1, s1, s3
+; GFX8-NEXT: s_add_i32 s2, s2, s1
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s2
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_v2i16:
@@ -2832,41 +3154,56 @@ define amdgpu_ps i32 @s_saddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: saddsat_v2i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX6-NEXT: s_ashr_i32 s1, s0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s1, v1
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s3, s0, 0
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_max_i32 s2, s0, 0
+; GFX6-NEXT: s_sub_i32 s3, 0x80000000, s3
+; GFX6-NEXT: s_sub_i32 s2, 0x7fffffff, s2
+; GFX6-NEXT: v_max_i32_e32 v0, s3, v0
+; GFX6-NEXT: v_min_i32_e32 v0, s2, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_med3_i32 v1, v1, s1, v2
-; GFX6-NEXT: v_med3_i32 v0, v0, s1, v2
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
+; GFX6-NEXT: s_min_i32 s2, s0, 0
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_max_i32 s1, s0, 0
+; GFX6-NEXT: s_sub_i32 s2, 0x80000000, s2
+; GFX6-NEXT: s_sub_i32 s1, 0x7fffffff, s1
+; GFX6-NEXT: v_max_i32_e32 v1, s2, v1
+; GFX6-NEXT: v_min_i32_e32 v1, s1, v1
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_v2i16_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_add_u16_e32 v2, s1, v1
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, s1, v2
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[2:3], 0, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, s[2:3], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v0
-; GFX8-NEXT: v_add_u16_e32 v0, s0, v0
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], s0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT: v_max_i16_e32 v1, s2, v0
+; GFX8-NEXT: s_sext_i32_i16 s2, s1
+; GFX8-NEXT: v_min_i16_e32 v1, s3, v1
+; GFX8-NEXT: s_max_i32 s3, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT: v_max_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v0, s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u16_e32 v1, s0, v1
+; GFX8-NEXT: v_add_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_v2i16_sv:
@@ -2886,44 +3223,55 @@ define amdgpu_ps float @saddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
define amdgpu_ps float @saddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: saddsat_v2i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s1, s0, 16
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s1, v1
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_med3_i32 v1, v1, s1, v2
-; GFX6-NEXT: v_med3_i32 v0, v0, s1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v4, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_max_i32_e32 v2, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v5, v4
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, 0x7fffffff, v2
+; GFX6-NEXT: v_max_i32_e32 v4, s0, v4
+; GFX6-NEXT: v_min_i32_e32 v2, v4, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_bfrev_b32_e32 v3, -2
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: v_max_i32_e32 v2, 0, v1
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v3, v2
+; GFX6-NEXT: v_min_i32_e32 v3, 0, v1
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT: v_max_i32_e32 v3, s0, v3
+; GFX6-NEXT: v_min_i32_e32 v2, v3, v2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: saddsat_v2i16_vs:
; GFX8: ; %bb.0:
+; GFX8-NEXT: v_min_i16_e32 v2, 0, v0
+; GFX8-NEXT: v_max_i16_e32 v1, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT: v_sub_u16_e32 v1, 0x7fff, v1
+; GFX8-NEXT: v_max_i16_e32 v2, s0, v2
+; GFX8-NEXT: v_min_i16_e32 v1, v2, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 0
+; GFX8-NEXT: v_max_i16_sdwa v3, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_add_u16_e32 v2, s1, v1
-; GFX8-NEXT: s_cmp_lt_i32 s0, 0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v2, v1
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, s[2:3], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX8-NEXT: v_add_u16_e32 v2, s0, v0
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s0, 0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v2, v0
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT: s_xor_b64 vcc, s[0:1], vcc
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT: v_sub_u16_e32 v3, 0x7fff, v3
+; GFX8-NEXT: v_max_i16_e32 v2, s1, v2
+; GFX8-NEXT: v_min_i16_e32 v2, v2, v3
+; GFX8-NEXT: v_add_u16_e32 v1, v0, v1
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: saddsat_v2i16_vs:
@@ -2955,71 +3303,96 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
; GFX6-LABEL: v_saddsat_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v2
-; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v0
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, v7, v6
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v10, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v11, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_max_i32_e32 v8, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, v11, v10
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, 0x7fffffff, v8
+; GFX6-NEXT: v_max_i32_e32 v2, v10, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v8
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_min_i32_e32 v8, 0, v2
+; GFX6-NEXT: v_bfrev_b32_e32 v9, -2
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_max_i32_e32 v6, 0, v2
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v11, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v9, v6
+; GFX6-NEXT: v_max_i32_e32 v4, v8, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v6
+; GFX6-NEXT: v_min_i32_e32 v6, 0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_max_i32_e32 v4, 0, v1
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v11, v6
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v9, v4
+; GFX6-NEXT: v_max_i32_e32 v3, v6, v3
+; GFX6-NEXT: v_min_i32_e32 v3, v3, v4
; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; GFX6-NEXT: v_med3_i32 v6, v6, s4, v7
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v7
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v7
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v7
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_min_i32_e32 v6, 0, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_max_i32_e32 v5, 0, v3
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v11, v6
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v9, v5
+; GFX6-NEXT: v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v5
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_add_u16_e32 v6, v5, v4
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v6, v5
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v6
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v2
-; GFX8-NEXT: v_add_u16_e32 v2, v0, v2
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v2, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_add_u16_e32 v5, v4, v2
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v5, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v5
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v3
-; GFX8-NEXT: v_add_u16_e32 v3, v1, v3
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v3, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v3
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v5, 0, v0
+; GFX8-NEXT: v_max_i16_e32 v4, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v5, 0x8000, v5
+; GFX8-NEXT: v_sub_u16_e32 v4, 0x7fff, v4
+; GFX8-NEXT: v_max_i16_e32 v5, v5, v2
+; GFX8-NEXT: v_min_i16_e32 v4, v5, v4
+; GFX8-NEXT: v_mov_b32_e32 v5, 0
+; GFX8-NEXT: v_min_i16_sdwa v7, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_sdwa v6, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_sub_u16_e32 v6, 0x7fff, v6
+; GFX8-NEXT: v_max_i16_sdwa v2, v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v7, 0, v1
+; GFX8-NEXT: v_min_i16_e32 v2, v2, v6
+; GFX8-NEXT: v_max_i16_e32 v6, 0, v1
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_sub_u16_e32 v6, 0x7fff, v6
+; GFX8-NEXT: v_max_i16_e32 v7, v7, v3
+; GFX8-NEXT: v_min_i16_e32 v6, v7, v6
+; GFX8-NEXT: v_max_i16_sdwa v7, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v5, 0x8000, v5
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x7fff, v7
+; GFX8-NEXT: v_max_i16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v3, v3, v7
+; GFX8-NEXT: v_add_u16_e32 v4, v0, v4
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v2, v1, v6
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v4i16:
@@ -3043,96 +3416,122 @@ define <2 x float> @v_saddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
define amdgpu_ps <2 x i32> @s_saddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs) {
; GFX6-LABEL: s_saddsat_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s6, s3, 16
-; GFX6-NEXT: s_ashr_i32 s7, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_add_i32 s7, s7, s6
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: s_add_i32 s1, s1, s3
-; GFX6-NEXT: s_ashr_i32 s4, s2, 16
-; GFX6-NEXT: s_ashr_i32 s5, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_med3_i32 v2, s7, v0, v1
-; GFX6-NEXT: v_med3_i32 v3, s1, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: s_add_i32 s5, s5, s4
+; GFX6-NEXT: s_lshr_b32 s4, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s9, s0, 0
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_max_i32 s8, s0, 0
+; GFX6-NEXT: s_sub_i32 s9, 0x80000000, s9
+; GFX6-NEXT: s_sub_i32 s8, 0x7fffffff, s8
+; GFX6-NEXT: s_max_i32 s2, s9, s2
+; GFX6-NEXT: s_min_i32 s2, s2, s8
; GFX6-NEXT: s_add_i32 s0, s0, s2
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_med3_i32 v3, s5, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s1, v2
+; GFX6-NEXT: s_lshl_b32 s2, s4, 16
+; GFX6-NEXT: s_min_i32 s8, s2, 0
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_max_i32 s6, s2, 0
+; GFX6-NEXT: s_sub_i32 s8, 0x80000000, s8
+; GFX6-NEXT: s_lshr_b32 s5, s1, 16
+; GFX6-NEXT: s_sub_i32 s6, 0x7fffffff, s6
+; GFX6-NEXT: s_max_i32 s4, s8, s4
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_min_i32 s4, s4, s6
+; GFX6-NEXT: s_min_i32 s6, s1, 0
+; GFX6-NEXT: s_lshr_b32 s7, s3, 16
+; GFX6-NEXT: s_add_i32 s2, s2, s4
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_max_i32 s4, s1, 0
+; GFX6-NEXT: s_sub_i32 s6, 0x80000000, s6
+; GFX6-NEXT: s_sub_i32 s4, 0x7fffffff, s4
+; GFX6-NEXT: s_max_i32 s3, s6, s3
+; GFX6-NEXT: s_min_i32 s3, s3, s4
+; GFX6-NEXT: s_add_i32 s1, s1, s3
+; GFX6-NEXT: s_lshl_b32 s3, s5, 16
+; GFX6-NEXT: s_min_i32 s6, s3, 0
+; GFX6-NEXT: s_lshl_b32 s4, s7, 16
+; GFX6-NEXT: s_max_i32 s5, s3, 0
+; GFX6-NEXT: s_sub_i32 s6, 0x80000000, s6
+; GFX6-NEXT: s_sub_i32 s5, 0x7fffffff, s5
+; GFX6-NEXT: s_max_i32 s4, s6, s4
+; GFX6-NEXT: s_ashr_i32 s2, s2, 16
+; GFX6-NEXT: s_min_i32 s4, s4, s5
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_add_i32 s3, s3, s4
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_ashr_i32 s3, s3, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s2
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_saddsat_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: s_add_i32 s8, s5, s4
-; GFX8-NEXT: s_sext_i32_i16 s9, s8
-; GFX8-NEXT: s_ashr_i32 s4, s1, 16
-; GFX8-NEXT: s_cmp_lt_i32 s9, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s3, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_xor_b64 s[4:5], s[6:7], s[4:5]
-; GFX8-NEXT: s_lshr_b32 s6, s9, 15
-; GFX8-NEXT: s_xor_b32 s6, s6, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s4, s6, s8
-; GFX8-NEXT: s_lshl_b32 s8, s4, 16
-; GFX8-NEXT: s_add_i32 s9, s1, s3
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_sext_i32_i16 s10, s9
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_cmp_lt_i32 s3, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, s1
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s10, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8-NEXT: s_lshr_b32 s3, s2, 16
+; GFX8-NEXT: s_sext_i32_i16 s8, s0
+; GFX8-NEXT: s_max_i32 s9, s8, 0
+; GFX8-NEXT: s_min_i32 s8, s8, 0
+; GFX8-NEXT: s_sub_i32 s8, 0x8000, s8
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sub_i32 s9, 0x7fff, s9
+; GFX8-NEXT: s_max_i32 s2, s8, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s8, s9
; GFX8-NEXT: s_lshr_b32 s4, s0, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_add_i32 s3, s4, s3
-; GFX8-NEXT: s_or_b32 s1, s1, s8
-; GFX8-NEXT: s_sext_i32_i16 s8, s3
-; GFX8-NEXT: s_ashr_i32 s4, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s8, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s2, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_xor_b64 s[4:5], s[6:7], s[4:5]
-; GFX8-NEXT: s_lshr_b32 s6, s8, 15
-; GFX8-NEXT: s_xor_b32 s6, s6, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s3, s6, s3
-; GFX8-NEXT: s_lshl_b32 s6, s3, 16
-; GFX8-NEXT: s_add_i32 s7, s0, s2
+; GFX8-NEXT: s_min_i32 s2, s2, s8
+; GFX8-NEXT: s_add_i32 s0, s0, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s4
+; GFX8-NEXT: s_max_i32 s8, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sub_i32 s8, 0x7fff, s8
+; GFX8-NEXT: s_max_i32 s2, s2, s6
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s6, s8
+; GFX8-NEXT: s_min_i32 s2, s2, s6
+; GFX8-NEXT: s_add_i32 s4, s4, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s1
+; GFX8-NEXT: s_max_i32 s6, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sub_i32 s6, 0x7fff, s6
+; GFX8-NEXT: s_max_i32 s2, s2, s3
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s3, s6
+; GFX8-NEXT: s_lshr_b32 s5, s1, 16
+; GFX8-NEXT: s_min_i32 s2, s2, s3
+; GFX8-NEXT: s_add_i32 s1, s1, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s5
+; GFX8-NEXT: s_max_i32 s3, s2, 0
+; GFX8-NEXT: s_min_i32 s2, s2, 0
+; GFX8-NEXT: s_sub_i32 s2, 0x8000, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s6, s7
+; GFX8-NEXT: s_sub_i32 s3, 0x7fff, s3
+; GFX8-NEXT: s_max_i32 s2, s2, s6
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_min_i32 s2, s2, s3
+; GFX8-NEXT: s_add_i32 s5, s5, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s4
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_sext_i32_i16 s8, s7
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s2, 0
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, s0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s8, 15
-; GFX8-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x8000
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX8-NEXT: s_cselect_b32 s0, s0, s7
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s6
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s5
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_v4i16:
@@ -3172,100 +3571,137 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
; GFX6-LABEL: v_saddsat_v6i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v8, 16, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v10, 16, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v11, 16, v0
-; GFX6-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_add_i32_e32 v10, vcc, v11, v10
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v14, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v15, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_max_i32_e32 v12, 0, v0
+; GFX6-NEXT: v_sub_i32_e32 v14, vcc, v15, v14
+; GFX6-NEXT: v_sub_i32_e32 v12, vcc, 0x7fffffff, v12
+; GFX6-NEXT: v_max_i32_e32 v3, v14, v3
+; GFX6-NEXT: v_min_i32_e32 v3, v3, v12
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v9, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_min_i32_e32 v12, 0, v3
+; GFX6-NEXT: v_bfrev_b32_e32 v13, -2
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT: v_max_i32_e32 v9, 0, v3
+; GFX6-NEXT: v_sub_i32_e32 v12, vcc, v15, v12
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v13, v9
+; GFX6-NEXT: v_max_i32_e32 v6, v12, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v9
+; GFX6-NEXT: v_min_i32_e32 v9, 0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_max_i32_e32 v6, 0, v1
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v15, v9
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v13, v6
+; GFX6-NEXT: v_max_i32_e32 v4, v9, v4
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v6
; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4
-; GFX6-NEXT: v_med3_i32 v3, v3, s4, v11
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v11
-; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v2
-; GFX6-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v7, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_min_i32_e32 v9, 0, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT: v_max_i32_e32 v7, 0, v4
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v15, v9
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_sub_i32_e32 v7, vcc, v13, v7
+; GFX6-NEXT: v_max_i32_e32 v6, v9, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v7
+; GFX6-NEXT: v_min_i32_e32 v7, 0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_max_i32_e32 v6, 0, v2
+; GFX6-NEXT: v_sub_i32_e32 v7, vcc, v15, v7
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v13, v6
+; GFX6-NEXT: v_max_i32_e32 v5, v7, v5
+; GFX6-NEXT: v_min_i32_e32 v5, v5, v6
; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5
-; GFX6-NEXT: v_med3_i32 v10, v10, s4, v11
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v11
-; GFX6-NEXT: v_med3_i32 v3, v3, s4, v11
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v11
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX6-NEXT: v_min_i32_e32 v8, 0, v5
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT: v_max_i32_e32 v7, 0, v5
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v15, v8
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_sub_i32_e32 v7, vcc, v13, v7
+; GFX6-NEXT: v_max_i32_e32 v6, v8, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v7
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v5
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v6i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX8-NEXT: v_add_u16_e32 v8, v7, v6
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v8, v7
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v6
-; GFX8-NEXT: v_ashrrev_i16_e32 v6, 15, v8
-; GFX8-NEXT: v_xor_b32_e32 v6, 0xffff8000, v6
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v3
-; GFX8-NEXT: v_add_u16_e32 v3, v0, v3
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v3, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v3
-; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_add_u16_e32 v7, v6, v3
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v7, v6
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 15, v7
-; GFX8-NEXT: v_xor_b32_e32 v3, 0xffff8000, v3
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v4
-; GFX8-NEXT: v_add_u16_e32 v4, v1, v4
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v4, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v4, v1, vcc
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v6, v4, v3
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v6, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 15, v6
-; GFX8-NEXT: v_xor_b32_e32 v3, 0xffff8000, v3
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_add_u16_e32 v4, v2, v5
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v5
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v4, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v7, 0, v0
+; GFX8-NEXT: v_max_i16_e32 v6, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_sub_u16_e32 v6, 0x7fff, v6
+; GFX8-NEXT: v_max_i16_e32 v7, v7, v3
+; GFX8-NEXT: v_min_i16_e32 v6, v7, v6
+; GFX8-NEXT: v_mov_b32_e32 v7, 0
+; GFX8-NEXT: v_min_i16_sdwa v9, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_sdwa v8, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_sub_u16_e32 v8, 0x7fff, v8
+; GFX8-NEXT: v_max_i16_sdwa v3, v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v9, 0, v1
+; GFX8-NEXT: v_min_i16_e32 v3, v3, v8
+; GFX8-NEXT: v_max_i16_e32 v8, 0, v1
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_sub_u16_e32 v8, 0x7fff, v8
+; GFX8-NEXT: v_max_i16_e32 v9, v9, v4
+; GFX8-NEXT: v_min_i16_sdwa v10, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v8, v9, v8
+; GFX8-NEXT: v_max_i16_sdwa v9, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v10, 0x8000, v10
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x7fff, v9
+; GFX8-NEXT: v_max_i16_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v10, 0, v2
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v9
+; GFX8-NEXT: v_max_i16_e32 v9, 0, v2
+; GFX8-NEXT: v_sub_u16_e32 v10, 0x8000, v10
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x7fff, v9
+; GFX8-NEXT: v_max_i16_e32 v10, v10, v5
+; GFX8-NEXT: v_min_i16_e32 v9, v10, v9
+; GFX8-NEXT: v_max_i16_sdwa v10, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_sdwa v7, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_sub_u16_e32 v10, 0x7fff, v10
+; GFX8-NEXT: v_max_i16_sdwa v5, v7, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v5, v5, v10
+; GFX8-NEXT: v_add_u16_e32 v6, v0, v6
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v3, v1, v8
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_add_u16_e32 v3, v2, v9
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v6i16:
@@ -3291,138 +3727,178 @@ define <3 x float> @v_saddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
define amdgpu_ps <3 x i32> @s_saddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inreg %rhs) {
; GFX6-LABEL: s_saddsat_v6i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s10, s5, 16
-; GFX6-NEXT: s_ashr_i32 s11, s2, 16
-; GFX6-NEXT: s_sext_i32_i16 s5, s5
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: s_add_i32 s11, s11, s10
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: s_add_i32 s2, s2, s5
-; GFX6-NEXT: s_ashr_i32 s8, s4, 16
-; GFX6-NEXT: s_ashr_i32 s9, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s4, s4
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: v_med3_i32 v2, s11, v0, v1
-; GFX6-NEXT: v_med3_i32 v3, s2, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: s_add_i32 s9, s9, s8
-; GFX6-NEXT: s_add_i32 s1, s1, s4
-; GFX6-NEXT: s_ashr_i32 s6, s3, 16
-; GFX6-NEXT: s_ashr_i32 s7, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_med3_i32 v3, s9, v0, v1
-; GFX6-NEXT: v_med3_i32 v4, s1, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX6-NEXT: s_add_i32 s7, s7, s6
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s13, s0, 0
+; GFX6-NEXT: s_lshr_b32 s9, s3, 16
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_max_i32 s12, s0, 0
+; GFX6-NEXT: s_sub_i32 s13, 0x80000000, s13
+; GFX6-NEXT: s_sub_i32 s12, 0x7fffffff, s12
+; GFX6-NEXT: s_max_i32 s3, s13, s3
+; GFX6-NEXT: s_min_i32 s3, s3, s12
; GFX6-NEXT: s_add_i32 s0, s0, s3
-; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX6-NEXT: v_med3_i32 v4, s7, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s1, v3
-; GFX6-NEXT: v_readfirstlane_b32 s2, v2
+; GFX6-NEXT: s_lshl_b32 s3, s6, 16
+; GFX6-NEXT: s_min_i32 s12, s3, 0
+; GFX6-NEXT: s_lshl_b32 s6, s9, 16
+; GFX6-NEXT: s_max_i32 s9, s3, 0
+; GFX6-NEXT: s_sub_i32 s12, 0x80000000, s12
+; GFX6-NEXT: s_lshr_b32 s7, s1, 16
+; GFX6-NEXT: s_sub_i32 s9, 0x7fffffff, s9
+; GFX6-NEXT: s_max_i32 s6, s12, s6
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_min_i32 s6, s6, s9
+; GFX6-NEXT: s_min_i32 s9, s1, 0
+; GFX6-NEXT: s_lshr_b32 s10, s4, 16
+; GFX6-NEXT: s_add_i32 s3, s3, s6
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_max_i32 s6, s1, 0
+; GFX6-NEXT: s_sub_i32 s9, 0x80000000, s9
+; GFX6-NEXT: s_sub_i32 s6, 0x7fffffff, s6
+; GFX6-NEXT: s_max_i32 s4, s9, s4
+; GFX6-NEXT: s_min_i32 s4, s4, s6
+; GFX6-NEXT: s_add_i32 s1, s1, s4
+; GFX6-NEXT: s_lshl_b32 s4, s7, 16
+; GFX6-NEXT: s_min_i32 s9, s4, 0
+; GFX6-NEXT: s_lshl_b32 s6, s10, 16
+; GFX6-NEXT: s_max_i32 s7, s4, 0
+; GFX6-NEXT: s_sub_i32 s9, 0x80000000, s9
+; GFX6-NEXT: s_lshr_b32 s8, s2, 16
+; GFX6-NEXT: s_sub_i32 s7, 0x7fffffff, s7
+; GFX6-NEXT: s_max_i32 s6, s9, s6
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_min_i32 s6, s6, s7
+; GFX6-NEXT: s_min_i32 s7, s2, 0
+; GFX6-NEXT: s_lshr_b32 s11, s5, 16
+; GFX6-NEXT: s_add_i32 s4, s4, s6
+; GFX6-NEXT: s_lshl_b32 s5, s5, 16
+; GFX6-NEXT: s_max_i32 s6, s2, 0
+; GFX6-NEXT: s_sub_i32 s7, 0x80000000, s7
+; GFX6-NEXT: s_sub_i32 s6, 0x7fffffff, s6
+; GFX6-NEXT: s_max_i32 s5, s7, s5
+; GFX6-NEXT: s_min_i32 s5, s5, s6
+; GFX6-NEXT: s_add_i32 s2, s2, s5
+; GFX6-NEXT: s_lshl_b32 s5, s8, 16
+; GFX6-NEXT: s_min_i32 s8, s5, 0
+; GFX6-NEXT: s_ashr_i32 s3, s3, 16
+; GFX6-NEXT: s_lshl_b32 s6, s11, 16
+; GFX6-NEXT: s_max_i32 s7, s5, 0
+; GFX6-NEXT: s_sub_i32 s8, 0x80000000, s8
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_sub_i32 s7, 0x7fffffff, s7
+; GFX6-NEXT: s_max_i32 s6, s8, s6
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_ashr_i32 s4, s4, 16
+; GFX6-NEXT: s_min_i32 s6, s6, s7
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_add_i32 s5, s5, s6
+; GFX6-NEXT: s_or_b32 s0, s0, s3
+; GFX6-NEXT: s_and_b32 s3, s4, 0xffff
+; GFX6-NEXT: s_ashr_i32 s5, s5, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_ashr_i32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s3
+; GFX6-NEXT: s_and_b32 s3, s5, 0xffff
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_saddsat_v6i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s2, 16
-; GFX8-NEXT: s_add_i32 s10, s7, s6
-; GFX8-NEXT: s_sext_i32_i16 s11, s10
-; GFX8-NEXT: s_ashr_i32 s6, s2, 16
-; GFX8-NEXT: s_cmp_lt_i32 s11, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s5, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[8:9], s[6:7]
-; GFX8-NEXT: s_lshr_b32 s8, s11, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s6, s8, s10
-; GFX8-NEXT: s_lshl_b32 s10, s6, 16
-; GFX8-NEXT: s_add_i32 s11, s2, s5
-; GFX8-NEXT: s_lshl_b32 s5, s5, 16
-; GFX8-NEXT: s_sext_i32_i16 s12, s11
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_cmp_lt_i32 s5, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s12, s2
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s12, 15
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s2, s2, s11
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_lshr_b32 s6, s1, 16
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_add_i32 s5, s6, s5
-; GFX8-NEXT: s_or_b32 s2, s2, s10
-; GFX8-NEXT: s_sext_i32_i16 s10, s5
-; GFX8-NEXT: s_ashr_i32 s6, s1, 16
-; GFX8-NEXT: s_cmp_lt_i32 s10, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s4, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[8:9], s[6:7]
-; GFX8-NEXT: s_lshr_b32 s8, s10, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s5, s8, s5
-; GFX8-NEXT: s_lshl_b32 s8, s5, 16
-; GFX8-NEXT: s_add_i32 s9, s1, s4
-; GFX8-NEXT: s_lshl_b32 s4, s4, 16
-; GFX8-NEXT: s_sext_i32_i16 s10, s9
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_cmp_lt_i32 s4, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, s1
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s10, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s0, 16
-; GFX8-NEXT: s_or_b32 s1, s1, s8
-; GFX8-NEXT: s_add_i32 s8, s5, s4
-; GFX8-NEXT: s_sext_i32_i16 s9, s8
-; GFX8-NEXT: s_ashr_i32 s4, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s9, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s3, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_xor_b64 s[4:5], s[6:7], s[4:5]
-; GFX8-NEXT: s_lshr_b32 s6, s9, 15
-; GFX8-NEXT: s_xor_b32 s6, s6, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s4, s6, s8
-; GFX8-NEXT: s_lshl_b32 s8, s4, 16
-; GFX8-NEXT: s_add_i32 s9, s0, s3
+; GFX8-NEXT: s_sext_i32_i16 s12, s0
+; GFX8-NEXT: s_max_i32 s13, s12, 0
+; GFX8-NEXT: s_min_i32 s12, s12, 0
+; GFX8-NEXT: s_sub_i32 s12, 0x8000, s12
+; GFX8-NEXT: s_lshr_b32 s9, s3, 16
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sub_i32 s13, 0x7fff, s13
+; GFX8-NEXT: s_max_i32 s3, s12, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s12, s13
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: s_min_i32 s3, s3, s12
+; GFX8-NEXT: s_add_i32 s0, s0, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s6
+; GFX8-NEXT: s_max_i32 s12, s3, 0
+; GFX8-NEXT: s_min_i32 s3, s3, 0
+; GFX8-NEXT: s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sub_i32 s12, 0x7fff, s12
+; GFX8-NEXT: s_max_i32 s3, s3, s9
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s9, s12
+; GFX8-NEXT: s_min_i32 s3, s3, s9
+; GFX8-NEXT: s_add_i32 s6, s6, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s1
+; GFX8-NEXT: s_max_i32 s9, s3, 0
+; GFX8-NEXT: s_min_i32 s3, s3, 0
+; GFX8-NEXT: s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT: s_lshr_b32 s10, s4, 16
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sub_i32 s9, 0x7fff, s9
+; GFX8-NEXT: s_max_i32 s3, s3, s4
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s4, s9
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: s_min_i32 s3, s3, s4
+; GFX8-NEXT: s_add_i32 s1, s1, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s7
+; GFX8-NEXT: s_max_i32 s4, s3, 0
+; GFX8-NEXT: s_min_i32 s3, s3, 0
+; GFX8-NEXT: s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s9, s10
+; GFX8-NEXT: s_sub_i32 s4, 0x7fff, s4
+; GFX8-NEXT: s_max_i32 s3, s3, s9
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_min_i32 s3, s3, s4
+; GFX8-NEXT: s_add_i32 s7, s7, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s2
+; GFX8-NEXT: s_max_i32 s4, s3, 0
+; GFX8-NEXT: s_min_i32 s3, s3, 0
+; GFX8-NEXT: s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT: s_lshr_b32 s11, s5, 16
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sub_i32 s4, 0x7fff, s4
+; GFX8-NEXT: s_max_i32 s3, s3, s5
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_lshr_b32 s8, s2, 16
+; GFX8-NEXT: s_min_i32 s3, s3, s4
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s8
+; GFX8-NEXT: s_max_i32 s4, s3, 0
+; GFX8-NEXT: s_min_i32 s3, s3, 0
+; GFX8-NEXT: s_sub_i32 s3, 0x8000, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s5, s11
+; GFX8-NEXT: s_sub_i32 s4, 0x7fff, s4
+; GFX8-NEXT: s_max_i32 s3, s3, s5
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_min_i32 s3, s3, s4
+; GFX8-NEXT: s_add_i32 s8, s8, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s6
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_sext_i32_i16 s10, s9
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s3, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, s0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s10, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s0, s0, s9
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s8
+; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s8
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_v6i16:
@@ -3456,129 +3932,178 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
; GFX6-LABEL: v_saddsat_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v12, 16, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v13, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v14, 16, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v15, 16, v0
-; GFX6-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_add_i32_e32 v14, vcc, v15, v14
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v15, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v18, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v19, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_max_i32_e32 v16, 0, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v17, -2
+; GFX6-NEXT: v_sub_i32_e32 v18, vcc, v19, v18
+; GFX6-NEXT: v_sub_i32_e32 v16, vcc, v17, v16
+; GFX6-NEXT: v_max_i32_e32 v4, v18, v4
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v16
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v4
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, v13, v12
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; GFX6-NEXT: v_min_i32_e32 v16, 0, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT: v_max_i32_e32 v12, 0, v4
+; GFX6-NEXT: v_sub_i32_e32 v16, vcc, v19, v16
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v12, vcc, v17, v12
+; GFX6-NEXT: v_max_i32_e32 v8, v16, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v12
+; GFX6-NEXT: v_min_i32_e32 v12, 0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_max_i32_e32 v8, 0, v1
+; GFX6-NEXT: v_sub_i32_e32 v12, vcc, v19, v12
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v17, v8
+; GFX6-NEXT: v_max_i32_e32 v5, v12, v5
+; GFX6-NEXT: v_min_i32_e32 v5, v5, v8
; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v5
-; GFX6-NEXT: v_med3_i32 v4, v4, s4, v15
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v15
-; GFX6-NEXT: v_ashrrev_i32_e32 v10, 16, v6
-; GFX6-NEXT: v_ashrrev_i32_e32 v11, 16, v2
-; GFX6-NEXT: v_bfe_i32 v6, v6, 0, 16
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v9
+; GFX6-NEXT: v_min_i32_e32 v12, 0, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT: v_max_i32_e32 v9, 0, v5
+; GFX6-NEXT: v_sub_i32_e32 v12, vcc, v19, v12
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT: v_max_i32_e32 v8, v12, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT: v_min_i32_e32 v9, 0, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_max_i32_e32 v8, 0, v2
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v19, v9
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v17, v8
+; GFX6-NEXT: v_max_i32_e32 v6, v9, v6
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v8
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT: v_min_i32_e32 v10, 0, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT: v_max_i32_e32 v9, 0, v6
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT: v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v9
+; GFX6-NEXT: v_min_i32_e32 v9, 0, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT: v_max_i32_e32 v8, 0, v3
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v19, v9
+; GFX6-NEXT: v_sub_i32_e32 v8, vcc, v17, v8
+; GFX6-NEXT: v_max_i32_e32 v7, v9, v7
+; GFX6-NEXT: v_min_i32_e32 v7, v7, v8
+; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v11
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_min_i32_e32 v10, 0, v7
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT: v_max_i32_e32 v9, 0, v7
+; GFX6-NEXT: v_sub_i32_e32 v10, vcc, v19, v10
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v9, vcc, v17, v9
+; GFX6-NEXT: v_max_i32_e32 v8, v10, v8
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v6
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v9
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, v11, v10
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v6
-; GFX6-NEXT: v_med3_i32 v4, v4, s4, v15
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v15
-; GFX6-NEXT: v_ashrrev_i32_e32 v8, 16, v7
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 16, v3
-; GFX6-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v7
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, v9, v8
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7
-; GFX6-NEXT: v_med3_i32 v14, v14, s4, v15
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v15
-; GFX6-NEXT: v_med3_i32 v4, v4, s4, v15
-; GFX6-NEXT: v_med3_i32 v3, v3, s4, v15
-; GFX6-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v14
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_saddsat_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX8-NEXT: v_add_u16_e32 v10, v9, v8
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v10, v9
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v8
-; GFX8-NEXT: v_ashrrev_i16_e32 v8, 15, v10
-; GFX8-NEXT: v_xor_b32_e32 v8, 0xffff8000, v8
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v4
-; GFX8-NEXT: v_add_u16_e32 v4, v0, v4
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v4, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v0, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v0, 0xffff8000, v0
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v4, v0, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX8-NEXT: v_add_u16_e32 v9, v8, v4
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v9, v8
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v9
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v9, v4, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v5
-; GFX8-NEXT: v_add_u16_e32 v5, v1, v5
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v5, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v5
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v8, v5, v4
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v8, v5
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v8
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_add_u16_e32 v5, v2, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v8, v4, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v6
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v5, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v5
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX8-NEXT: v_add_u16_e32 v6, v5, v4
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v6, v5
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v6
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, s[4:5], vcc
-; GFX8-NEXT: v_add_u16_e32 v5, v3, v7
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, 0, v7
-; GFX8-NEXT: v_cmp_lt_i16_e64 s[4:5], v5, v3
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 15, v5
-; GFX8-NEXT: v_xor_b32_e32 v3, 0xffff8000, v3
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v9, 0, v0
+; GFX8-NEXT: v_max_i16_e32 v8, 0, v0
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_sub_u16_e32 v8, 0x7fff, v8
+; GFX8-NEXT: v_max_i16_e32 v9, v9, v4
+; GFX8-NEXT: v_min_i16_e32 v8, v9, v8
+; GFX8-NEXT: v_mov_b32_e32 v9, 0
+; GFX8-NEXT: v_min_i16_sdwa v11, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_sdwa v10, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v11, 0x8000, v11
+; GFX8-NEXT: v_sub_u16_e32 v10, 0x7fff, v10
+; GFX8-NEXT: v_max_i16_sdwa v4, v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v11, 0, v1
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v10
+; GFX8-NEXT: v_max_i16_e32 v10, 0, v1
+; GFX8-NEXT: v_sub_u16_e32 v11, 0x8000, v11
+; GFX8-NEXT: v_sub_u16_e32 v10, 0x7fff, v10
+; GFX8-NEXT: v_max_i16_e32 v11, v11, v5
+; GFX8-NEXT: v_min_i16_sdwa v12, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v10, v11, v10
+; GFX8-NEXT: v_max_i16_sdwa v11, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v12, 0x8000, v12
+; GFX8-NEXT: v_sub_u16_e32 v11, 0x7fff, v11
+; GFX8-NEXT: v_max_i16_sdwa v5, v12, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v12, 0, v2
+; GFX8-NEXT: v_min_i16_e32 v5, v5, v11
+; GFX8-NEXT: v_max_i16_e32 v11, 0, v2
+; GFX8-NEXT: v_sub_u16_e32 v12, 0x8000, v12
+; GFX8-NEXT: v_sub_u16_e32 v11, 0x7fff, v11
+; GFX8-NEXT: v_max_i16_e32 v12, v12, v6
+; GFX8-NEXT: v_min_i16_sdwa v13, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v11, v12, v11
+; GFX8-NEXT: v_max_i16_sdwa v12, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v13, 0x8000, v13
+; GFX8-NEXT: v_sub_u16_e32 v12, 0x7fff, v12
+; GFX8-NEXT: v_max_i16_sdwa v6, v13, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v13, 0, v3
+; GFX8-NEXT: v_min_i16_e32 v6, v6, v12
+; GFX8-NEXT: v_max_i16_e32 v12, 0, v3
+; GFX8-NEXT: v_sub_u16_e32 v13, 0x8000, v13
+; GFX8-NEXT: v_sub_u16_e32 v12, 0x7fff, v12
+; GFX8-NEXT: v_max_i16_e32 v13, v13, v7
+; GFX8-NEXT: v_min_i16_e32 v12, v13, v12
+; GFX8-NEXT: v_max_i16_sdwa v13, v3, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_sdwa v9, v3, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_sub_u16_e32 v13, 0x7fff, v13
+; GFX8-NEXT: v_max_i16_sdwa v7, v9, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e32 v8, v0, v8
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v4, v1, v10
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v7, v7, v13
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_add_u16_e32 v4, v2, v11
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_add_u16_e32 v4, v3, v12
+; GFX8-NEXT: v_add_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_saddsat_v8i16:
@@ -3606,180 +4131,234 @@ define <4 x float> @v_saddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
define amdgpu_ps <4 x i32> @s_saddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inreg %rhs) {
; GFX6-LABEL: s_saddsat_v8i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s14, s7, 16
-; GFX6-NEXT: s_ashr_i32 s15, s3, 16
-; GFX6-NEXT: s_sext_i32_i16 s7, s7
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_add_i32 s15, s15, s14
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: s_add_i32 s3, s3, s7
-; GFX6-NEXT: s_ashr_i32 s12, s6, 16
-; GFX6-NEXT: s_ashr_i32 s13, s2, 16
-; GFX6-NEXT: s_sext_i32_i16 s6, s6
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: v_med3_i32 v2, s15, v0, v1
-; GFX6-NEXT: v_med3_i32 v3, s3, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: s_add_i32 s13, s13, s12
-; GFX6-NEXT: s_add_i32 s2, s2, s6
-; GFX6-NEXT: s_ashr_i32 s10, s5, 16
-; GFX6-NEXT: s_ashr_i32 s11, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s5, s5
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_med3_i32 v3, s13, v0, v1
-; GFX6-NEXT: v_med3_i32 v4, s2, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX6-NEXT: s_add_i32 s11, s11, s10
-; GFX6-NEXT: s_add_i32 s1, s1, s5
-; GFX6-NEXT: s_ashr_i32 s8, s4, 16
-; GFX6-NEXT: s_ashr_i32 s9, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s4, s4
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX6-NEXT: v_med3_i32 v4, s11, v0, v1
-; GFX6-NEXT: v_med3_i32 v5, s1, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX6-NEXT: s_add_i32 s9, s9, s8
+; GFX6-NEXT: s_lshr_b32 s8, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s17, s0, 0
+; GFX6-NEXT: s_lshr_b32 s12, s4, 16
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_max_i32 s16, s0, 0
+; GFX6-NEXT: s_sub_i32 s17, 0x80000000, s17
+; GFX6-NEXT: s_sub_i32 s16, 0x7fffffff, s16
+; GFX6-NEXT: s_max_i32 s4, s17, s4
+; GFX6-NEXT: s_min_i32 s4, s4, s16
; GFX6-NEXT: s_add_i32 s0, s0, s4
-; GFX6-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX6-NEXT: v_med3_i32 v5, s9, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s1, v4
-; GFX6-NEXT: v_readfirstlane_b32 s2, v3
-; GFX6-NEXT: v_readfirstlane_b32 s3, v2
+; GFX6-NEXT: s_lshl_b32 s4, s8, 16
+; GFX6-NEXT: s_min_i32 s16, s4, 0
+; GFX6-NEXT: s_lshl_b32 s8, s12, 16
+; GFX6-NEXT: s_max_i32 s12, s4, 0
+; GFX6-NEXT: s_sub_i32 s16, 0x80000000, s16
+; GFX6-NEXT: s_lshr_b32 s9, s1, 16
+; GFX6-NEXT: s_sub_i32 s12, 0x7fffffff, s12
+; GFX6-NEXT: s_max_i32 s8, s16, s8
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s12
+; GFX6-NEXT: s_min_i32 s12, s1, 0
+; GFX6-NEXT: s_lshr_b32 s13, s5, 16
+; GFX6-NEXT: s_add_i32 s4, s4, s8
+; GFX6-NEXT: s_lshl_b32 s5, s5, 16
+; GFX6-NEXT: s_max_i32 s8, s1, 0
+; GFX6-NEXT: s_sub_i32 s12, 0x80000000, s12
+; GFX6-NEXT: s_sub_i32 s8, 0x7fffffff, s8
+; GFX6-NEXT: s_max_i32 s5, s12, s5
+; GFX6-NEXT: s_min_i32 s5, s5, s8
+; GFX6-NEXT: s_add_i32 s1, s1, s5
+; GFX6-NEXT: s_lshl_b32 s5, s9, 16
+; GFX6-NEXT: s_min_i32 s12, s5, 0
+; GFX6-NEXT: s_lshl_b32 s8, s13, 16
+; GFX6-NEXT: s_max_i32 s9, s5, 0
+; GFX6-NEXT: s_sub_i32 s12, 0x80000000, s12
+; GFX6-NEXT: s_lshr_b32 s10, s2, 16
+; GFX6-NEXT: s_sub_i32 s9, 0x7fffffff, s9
+; GFX6-NEXT: s_max_i32 s8, s12, s8
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s9
+; GFX6-NEXT: s_min_i32 s9, s2, 0
+; GFX6-NEXT: s_lshr_b32 s14, s6, 16
+; GFX6-NEXT: s_add_i32 s5, s5, s8
+; GFX6-NEXT: s_lshl_b32 s6, s6, 16
+; GFX6-NEXT: s_max_i32 s8, s2, 0
+; GFX6-NEXT: s_sub_i32 s9, 0x80000000, s9
+; GFX6-NEXT: s_sub_i32 s8, 0x7fffffff, s8
+; GFX6-NEXT: s_max_i32 s6, s9, s6
+; GFX6-NEXT: s_min_i32 s6, s6, s8
+; GFX6-NEXT: s_add_i32 s2, s2, s6
+; GFX6-NEXT: s_lshl_b32 s6, s10, 16
+; GFX6-NEXT: s_min_i32 s10, s6, 0
+; GFX6-NEXT: s_lshl_b32 s8, s14, 16
+; GFX6-NEXT: s_max_i32 s9, s6, 0
+; GFX6-NEXT: s_sub_i32 s10, 0x80000000, s10
+; GFX6-NEXT: s_lshr_b32 s11, s3, 16
+; GFX6-NEXT: s_sub_i32 s9, 0x7fffffff, s9
+; GFX6-NEXT: s_max_i32 s8, s10, s8
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s9
+; GFX6-NEXT: s_min_i32 s9, s3, 0
+; GFX6-NEXT: s_lshr_b32 s15, s7, 16
+; GFX6-NEXT: s_add_i32 s6, s6, s8
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_max_i32 s8, s3, 0
+; GFX6-NEXT: s_sub_i32 s9, 0x80000000, s9
+; GFX6-NEXT: s_sub_i32 s8, 0x7fffffff, s8
+; GFX6-NEXT: s_max_i32 s7, s9, s7
+; GFX6-NEXT: s_min_i32 s7, s7, s8
+; GFX6-NEXT: s_ashr_i32 s4, s4, 16
+; GFX6-NEXT: s_add_i32 s3, s3, s7
+; GFX6-NEXT: s_lshl_b32 s7, s11, 16
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_min_i32 s10, s7, 0
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_ashr_i32 s5, s5, 16
+; GFX6-NEXT: s_lshl_b32 s8, s15, 16
+; GFX6-NEXT: s_max_i32 s9, s7, 0
+; GFX6-NEXT: s_sub_i32 s10, 0x80000000, s10
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_sub_i32 s9, 0x7fffffff, s9
+; GFX6-NEXT: s_max_i32 s8, s10, s8
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX6-NEXT: s_ashr_i32 s6, s6, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s9
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_ashr_i32 s2, s2, 16
+; GFX6-NEXT: s_add_i32 s7, s7, s8
+; GFX6-NEXT: s_or_b32 s1, s1, s4
+; GFX6-NEXT: s_and_b32 s4, s6, 0xffff
+; GFX6-NEXT: s_ashr_i32 s7, s7, 16
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_ashr_i32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_or_b32 s3, s3, s4
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_saddsat_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s8, s7, 16
-; GFX8-NEXT: s_lshr_b32 s9, s3, 16
-; GFX8-NEXT: s_add_i32 s12, s9, s8
-; GFX8-NEXT: s_sext_i32_i16 s13, s12
-; GFX8-NEXT: s_ashr_i32 s8, s3, 16
-; GFX8-NEXT: s_cmp_lt_i32 s13, s8
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s7, 0
-; GFX8-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX8-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
-; GFX8-NEXT: s_lshr_b32 s10, s13, 15
-; GFX8-NEXT: s_xor_b32 s10, s10, 0x8000
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT: s_cselect_b32 s8, s10, s12
-; GFX8-NEXT: s_lshl_b32 s12, s8, 16
-; GFX8-NEXT: s_add_i32 s13, s3, s7
-; GFX8-NEXT: s_lshl_b32 s7, s7, 16
-; GFX8-NEXT: s_sext_i32_i16 s14, s13
-; GFX8-NEXT: s_sext_i32_i16 s3, s3
-; GFX8-NEXT: s_cmp_lt_i32 s7, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s14, s3
-; GFX8-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX8-NEXT: s_lshr_b32 s3, s14, 15
-; GFX8-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; GFX8-NEXT: s_xor_b32 s3, s3, 0x8000
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT: s_cselect_b32 s3, s3, s13
-; GFX8-NEXT: s_lshr_b32 s7, s6, 16
-; GFX8-NEXT: s_lshr_b32 s8, s2, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_add_i32 s7, s8, s7
-; GFX8-NEXT: s_or_b32 s3, s3, s12
-; GFX8-NEXT: s_sext_i32_i16 s12, s7
-; GFX8-NEXT: s_ashr_i32 s8, s2, 16
-; GFX8-NEXT: s_cmp_lt_i32 s12, s8
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX8-NEXT: s_xor_b64 s[8:9], s[10:11], s[8:9]
-; GFX8-NEXT: s_lshr_b32 s10, s12, 15
-; GFX8-NEXT: s_xor_b32 s10, s10, 0x8000
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT: s_cselect_b32 s7, s10, s7
-; GFX8-NEXT: s_lshl_b32 s10, s7, 16
-; GFX8-NEXT: s_add_i32 s11, s2, s6
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_sext_i32_i16 s12, s11
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s12, s2
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s12, 15
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s2, s2, s11
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s1, 16
-; GFX8-NEXT: s_or_b32 s2, s2, s10
-; GFX8-NEXT: s_add_i32 s10, s7, s6
-; GFX8-NEXT: s_sext_i32_i16 s11, s10
-; GFX8-NEXT: s_ashr_i32 s6, s1, 16
-; GFX8-NEXT: s_cmp_lt_i32 s11, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s5, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[8:9], s[6:7]
-; GFX8-NEXT: s_lshr_b32 s8, s11, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s6, s8, s10
-; GFX8-NEXT: s_lshl_b32 s10, s6, 16
-; GFX8-NEXT: s_add_i32 s11, s1, s5
-; GFX8-NEXT: s_lshl_b32 s5, s5, 16
-; GFX8-NEXT: s_sext_i32_i16 s12, s11
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_cmp_lt_i32 s5, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s12, s1
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s12, 15
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s11
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_lshr_b32 s6, s0, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_add_i32 s5, s6, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s10
-; GFX8-NEXT: s_sext_i32_i16 s10, s5
-; GFX8-NEXT: s_ashr_i32 s6, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s10, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s4, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[8:9], s[6:7]
-; GFX8-NEXT: s_lshr_b32 s8, s10, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s5, s8, s5
-; GFX8-NEXT: s_lshl_b32 s8, s5, 16
-; GFX8-NEXT: s_add_i32 s9, s0, s4
+; GFX8-NEXT: s_sext_i32_i16 s16, s0
+; GFX8-NEXT: s_max_i32 s17, s16, 0
+; GFX8-NEXT: s_min_i32 s16, s16, 0
+; GFX8-NEXT: s_sub_i32 s16, 0x8000, s16
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
+; GFX8-NEXT: s_sext_i32_i16 s16, s16
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sub_i32 s17, 0x7fff, s17
+; GFX8-NEXT: s_max_i32 s4, s16, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s16, s17
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
+; GFX8-NEXT: s_min_i32 s4, s4, s16
+; GFX8-NEXT: s_add_i32 s0, s0, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s8
+; GFX8-NEXT: s_max_i32 s16, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_sub_i32 s16, 0x7fff, s16
+; GFX8-NEXT: s_max_i32 s4, s4, s12
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s12, s16
+; GFX8-NEXT: s_min_i32 s4, s4, s12
+; GFX8-NEXT: s_add_i32 s8, s8, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s1
+; GFX8-NEXT: s_max_i32 s12, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sub_i32 s12, 0x7fff, s12
+; GFX8-NEXT: s_max_i32 s4, s4, s5
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s12
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_add_i32 s1, s1, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s9
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s12, s13
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s4, s4, s12
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_add_i32 s9, s9, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s2
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s4, s4, s6
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_add_i32 s2, s2, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s10
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s14
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s4, s4, s6
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_add_i32 s10, s10, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s3
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s7
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s4, s4, s6
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_add_i32 s3, s3, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s11
+; GFX8-NEXT: s_max_i32 s5, s4, 0
+; GFX8-NEXT: s_min_i32 s4, s4, 0
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: s_sub_i32 s4, 0x8000, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s15
+; GFX8-NEXT: s_sub_i32 s5, 0x7fff, s5
+; GFX8-NEXT: s_max_i32 s4, s4, s6
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_add_i32 s11, s11, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s8
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
; GFX8-NEXT: s_lshl_b32 s4, s4, 16
-; GFX8-NEXT: s_sext_i32_i16 s10, s9
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s4, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, s0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s10, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s0, s0, s9
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s8
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s9
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s10
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s11
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_saddsat_v8i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index da4093d80e04f..61bd7cb51f0d0 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; xUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu8.02-amd-amdhsa < %s | FileCheck --check-prefix=GFX8 %s
+; RUN: llc -global-isel -global-isel-abort=2 -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu8.02-amd-amdhsa < %s | FileCheck --check-prefix=GFX8 %s
; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10 %s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
index 5979289b56e06..c94a950c85dde 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sext_inreg.ll
@@ -546,22 +546,21 @@ define <2 x i16> @v_sext_inreg_v2i16_8(<2 x i16> %value) {
; GFX6-LABEL: v_sext_inreg_v2i16_8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfe_i32 v1, v0, 16, 8
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_bfe_i32 v1, v0, 0, 8
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 8
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sext_inreg_v2i16_8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, 0xffff
-; GFX8-NEXT: v_and_b32_sdwa v1, v1, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, 16
-; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, sext(v0) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_and_b32_sdwa v2, sext(v0), v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_sdwa v0, sext(v0), v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_2 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sext_inreg_v2i16_8:
@@ -586,19 +585,20 @@ define <2 x i16> @v_sext_inreg_v2i16_15(<2 x i16> %value) {
; GFX6-LABEL: v_sext_inreg_v2i16_15:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfe_i32 v1, v0, 16, 1
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_bfe_i32 v1, v0, 0, 1
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sext_inreg_v2i16_15:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_bfe_i32 v1, v0, 0, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 1
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
@@ -624,23 +624,22 @@ define <2 x i16> @v_sext_inreg_v2i16_15(<2 x i16> %value) {
define amdgpu_ps i32 @s_sext_inreg_v2i16_11(<2 x i16> inreg %value) {
; GFX6-LABEL: s_sext_inreg_v2i16_11:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_bfe_i32 s1, s0, 0x50010
-; GFX6-NEXT: s_bfe_i32 s0, s0, 0x50000
-; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_bfe_i32 s1, s0, 0x50000
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x50010
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_or_b32 s0, s1, s0
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sext_inreg_v2i16_11:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_u32 s1, s0, 0x100010
-; GFX8-NEXT: s_lshl_b32 s1, s1, 11
-; GFX8-NEXT: s_lshl_b32 s0, s0, 11
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_lshl_b32 s1, s1, 5
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x10000b
-; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_bfe_i32 s1, s0, 0x50000
+; GFX8-NEXT: s_bfe_i32 s0, s0, 0x50010
+; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_or_b32 s0, s1, s0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_sext_inreg_v2i16_11:
@@ -693,32 +692,33 @@ define <2 x float> @v_sext_inreg_v4i16_3(<4 x i16> %value) {
; GFX6-LABEL: v_sext_inreg_v4i16_3:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfe_i32 v2, v0, 16, 13
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 13
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_bfe_i32 v2, v0, 0, 13
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 13
+; GFX6-NEXT: v_bfe_i32 v3, v1, 0, 13
+; GFX6-NEXT: v_bfe_i32 v1, v1, 16, 13
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_bfe_i32 v2, v1, 16, 13
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 13
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sext_inreg_v4i16_3:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 3
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 3, v0
-; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 3, v2
-; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, 3, v1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 3, v2
-; GFX8-NEXT: v_ashrrev_i16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX8-NEXT: v_bfe_i32 v2, v0, 0, 13
+; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 13
+; GFX8-NEXT: v_bfe_i32 v3, v1, 0, 13
+; GFX8-NEXT: v_bfe_i32 v1, v1, 16, 13
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sext_inreg_v4i16_3:
@@ -747,36 +747,34 @@ define <2 x float> @v_sext_inreg_v4i16_3(<4 x i16> %value) {
define amdgpu_ps <2 x i32> @s_sext_inreg_v4i16_14(<4 x i16> inreg %value) {
; GFX6-LABEL: s_sext_inreg_v4i16_14:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_bfe_i32 s2, s1, 0x20010
-; GFX6-NEXT: s_bfe_i32 s1, s1, 0x20000
-; GFX6-NEXT: s_lshl_b32 s2, s2, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_or_b32 s1, s1, s2
-; GFX6-NEXT: s_bfe_i32 s2, s0, 0x20010
-; GFX6-NEXT: s_bfe_i32 s0, s0, 0x20000
-; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_bfe_i32 s2, s0, 0x20000
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0x20010
+; GFX6-NEXT: s_bfe_i32 s3, s1, 0x20000
+; GFX6-NEXT: s_bfe_i32 s1, s1, 0x20010
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_or_b32 s0, s2, s0
+; GFX6-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s1, s2, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sext_inreg_v4i16_14:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_u32 s2, s1, 0x100010
-; GFX8-NEXT: s_lshl_b32 s2, s2, 14
-; GFX8-NEXT: s_lshl_b32 s1, s1, 14
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 2
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x10000e
-; GFX8-NEXT: s_or_b32 s1, s1, s2
-; GFX8-NEXT: s_bfe_u32 s2, s0, 0x100010
-; GFX8-NEXT: s_lshl_b32 s2, s2, 14
-; GFX8-NEXT: s_lshl_b32 s0, s0, 14
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_lshl_b32 s2, s2, 2
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x10000e
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_bfe_i32 s2, s0, 0x20000
+; GFX8-NEXT: s_bfe_i32 s0, s0, 0x20010
+; GFX8-NEXT: s_bfe_i32 s3, s1, 0x20000
+; GFX8-NEXT: s_bfe_i32 s1, s1, 0x20010
+; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-NEXT: s_or_b32 s0, s2, s0
+; GFX8-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s1, s2, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_sext_inreg_v4i16_14:
@@ -857,52 +855,55 @@ define <4 x float> @v_sext_inreg_v8i16_11(<8 x i16> %value) {
; GFX6-LABEL: v_sext_inreg_v8i16_11:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_bfe_i32 v4, v0, 16, 5
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 5
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_bfe_i32 v4, v0, 0, 5
+; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 5
+; GFX6-NEXT: v_bfe_i32 v5, v1, 0, 5
+; GFX6-NEXT: v_bfe_i32 v1, v1, 16, 5
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX6-NEXT: v_bfe_i32 v4, v1, 16, 5
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 5
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_bfe_i32 v6, v2, 0, 5
+; GFX6-NEXT: v_bfe_i32 v2, v2, 16, 5
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX6-NEXT: v_bfe_i32 v4, v2, 16, 5
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 5
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_bfe_i32 v7, v3, 0, 5
+; GFX6-NEXT: v_bfe_i32 v3, v3, 16, 5
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_bfe_i32 v4, v3, 16, 5
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 5
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_sext_inreg_v8i16_11:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, 11
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 11, v0
-; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 11, v4
-; GFX8-NEXT: v_ashrrev_i16_sdwa v0, v5, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 11, v1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 11, v4
-; GFX8-NEXT: v_ashrrev_i16_sdwa v1, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 11, v2
-; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 11, v4
-; GFX8-NEXT: v_ashrrev_i16_sdwa v2, v5, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
-; GFX8-NEXT: v_lshlrev_b16_e32 v4, 11, v3
-; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 11, v4
-; GFX8-NEXT: v_ashrrev_i16_sdwa v3, v5, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX8-NEXT: v_bfe_i32 v4, v0, 0, 5
+; GFX8-NEXT: v_bfe_i32 v0, v0, 16, 5
+; GFX8-NEXT: v_bfe_i32 v5, v1, 0, 5
+; GFX8-NEXT: v_bfe_i32 v1, v1, 16, 5
+; GFX8-NEXT: v_bfe_i32 v6, v2, 0, 5
+; GFX8-NEXT: v_bfe_i32 v2, v2, 16, 5
+; GFX8-NEXT: v_bfe_i32 v7, v3, 0, 5
+; GFX8-NEXT: v_bfe_i32 v3, v3, 16, 5
+; GFX8-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX8-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sext_inreg_v8i16_11:
@@ -939,62 +940,58 @@ define <4 x float> @v_sext_inreg_v8i16_11(<8 x i16> %value) {
define amdgpu_ps <4 x i32> @s_sext_inreg_v8i16_5(<8 x i16> inreg %value) {
; GFX6-LABEL: s_sext_inreg_v8i16_5:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_bfe_i32 s4, s3, 0xb0010
-; GFX6-NEXT: s_bfe_i32 s3, s3, 0xb0000
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
-; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX6-NEXT: s_or_b32 s3, s3, s4
-; GFX6-NEXT: s_bfe_i32 s4, s2, 0xb0010
-; GFX6-NEXT: s_bfe_i32 s2, s2, 0xb0000
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
-; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX6-NEXT: s_or_b32 s2, s2, s4
-; GFX6-NEXT: s_bfe_i32 s4, s1, 0xb0010
-; GFX6-NEXT: s_bfe_i32 s1, s1, 0xb0000
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_or_b32 s1, s1, s4
-; GFX6-NEXT: s_bfe_i32 s4, s0, 0xb0010
-; GFX6-NEXT: s_bfe_i32 s0, s0, 0xb0000
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_bfe_i32 s4, s0, 0xb0000
+; GFX6-NEXT: s_bfe_i32 s0, s0, 0xb0010
+; GFX6-NEXT: s_bfe_i32 s5, s1, 0xb0000
+; GFX6-NEXT: s_bfe_i32 s1, s1, 0xb0010
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_bfe_i32 s6, s2, 0xb0000
+; GFX6-NEXT: s_bfe_i32 s2, s2, 0xb0010
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_bfe_i32 s7, s3, 0xb0000
+; GFX6-NEXT: s_bfe_i32 s3, s3, 0xb0010
+; GFX6-NEXT: s_or_b32 s0, s4, s0
+; GFX6-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_or_b32 s1, s4, s1
+; GFX6-NEXT: s_and_b32 s4, s6, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_or_b32 s2, s4, s2
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s3, s4, s3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_sext_inreg_v8i16_5:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_bfe_u32 s4, s3, 0x100010
-; GFX8-NEXT: s_lshl_b32 s4, s4, 5
-; GFX8-NEXT: s_lshl_b32 s3, s3, 5
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sext_i32_i16 s3, s3
-; GFX8-NEXT: s_lshl_b32 s4, s4, 11
-; GFX8-NEXT: s_bfe_u32 s3, s3, 0x100005
-; GFX8-NEXT: s_or_b32 s3, s3, s4
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x100010
-; GFX8-NEXT: s_lshl_b32 s4, s4, 5
-; GFX8-NEXT: s_lshl_b32 s2, s2, 5
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_lshl_b32 s4, s4, 11
-; GFX8-NEXT: s_bfe_u32 s2, s2, 0x100005
-; GFX8-NEXT: s_or_b32 s2, s2, s4
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x100010
-; GFX8-NEXT: s_lshl_b32 s4, s4, 5
-; GFX8-NEXT: s_lshl_b32 s1, s1, 5
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_lshl_b32 s4, s4, 11
-; GFX8-NEXT: s_bfe_u32 s1, s1, 0x100005
-; GFX8-NEXT: s_or_b32 s1, s1, s4
-; GFX8-NEXT: s_bfe_u32 s4, s0, 0x100010
-; GFX8-NEXT: s_lshl_b32 s4, s4, 5
-; GFX8-NEXT: s_lshl_b32 s0, s0, 5
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_lshl_b32 s4, s4, 11
-; GFX8-NEXT: s_bfe_u32 s0, s0, 0x100005
-; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_bfe_i32 s4, s0, 0xb0000
+; GFX8-NEXT: s_bfe_i32 s0, s0, 0xb0010
+; GFX8-NEXT: s_bfe_i32 s5, s1, 0xb0000
+; GFX8-NEXT: s_bfe_i32 s1, s1, 0xb0010
+; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX8-NEXT: s_bfe_i32 s6, s2, 0xb0000
+; GFX8-NEXT: s_bfe_i32 s2, s2, 0xb0010
+; GFX8-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX8-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX8-NEXT: s_bfe_i32 s7, s3, 0xb0000
+; GFX8-NEXT: s_bfe_i32 s3, s3, 0xb0010
+; GFX8-NEXT: s_or_b32 s0, s4, s0
+; GFX8-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-NEXT: s_or_b32 s1, s4, s1
+; GFX8-NEXT: s_and_b32 s4, s6, 0xffff
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX8-NEXT: s_or_b32 s2, s4, s2
+; GFX8-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s3, s4, s3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_sext_inreg_v8i16_5:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
index a5fbd0a99d627..17667e2abdb7f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/shl.ll
@@ -705,19 +705,21 @@ define <2 x i16> @v_shl_v2i16(<2 x i16> %value, <2 x i16> %amount) {
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, v1, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, v3, v2
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v1, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, v1, v0
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_v2i16:
@@ -739,17 +741,20 @@ define <2 x i16> @v_shl_v2i16_15(<2 x i16> %value) {
; GFX6-LABEL: v_shl_v2i16_15:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v0, 15, v0
-; GFX6-NEXT: v_and_b32_e32 v0, 0x80008000, v0
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 31, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_v2i16_15:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 15, v0
-; GFX8-NEXT: v_and_b32_e32 v1, 0x80000000, v1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, 15, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, 15
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, 15, v0
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_v2i16_15:
@@ -770,24 +775,26 @@ define <2 x i16> @v_shl_v2i16_15(<2 x i16> %value) {
define amdgpu_ps i32 @s_shl_v2i16(<2 x i16> inreg %value, <2 x i16> inreg %amount) {
; GFX6-LABEL: s_shl_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s2, s1, 16
-; GFX6-NEXT: s_lshr_b32 s3, s0, 16
-; GFX6-NEXT: s_lshl_b32 s2, s3, s2
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
; GFX6-NEXT: s_lshl_b32 s0, s0, s1
-; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_lshl_b32 s1, s2, s3
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_shl_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_lshl_b32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_lshl_b32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_lshl_b32 s1, s2, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_shl_v2i16:
@@ -817,20 +824,22 @@ define amdgpu_ps float @shl_v2i16_sv(<2 x i16> inreg %value, <2 x i16> %amount)
; GFX6: ; %bb.0:
; GFX6-NEXT: s_lshr_b32 s1, s0, 16
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshl_b32_e32 v1, s1, v1
; GFX6-NEXT: v_lshl_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: shl_v2i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshlrev_b16_e64 v0, v0, s0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_lshlrev_b16_e64 v1, v0, s0
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: shl_v2i16_sv:
@@ -852,20 +861,22 @@ define amdgpu_ps float @shl_v2i16_vs(<2 x i16> %value, <2 x i16> inreg %amount)
; GFX6: ; %bb.0:
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
; GFX6-NEXT: v_lshlrev_b32_e32 v1, s1, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: shl_v2i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, s0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, s0, v0
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: shl_v2i16_vs:
@@ -897,31 +908,35 @@ define <2 x float> @v_shl_v4i16(<4 x i16> %value, <4 x i16> %amount) {
; GFX6-LABEL: v_shl_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, v7, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, v5, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v1, v3, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, v7, v5
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b16_sdwa v4, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v2, v0
-; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, v3, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, v2, v0
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, v3, v1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_v4i16:
@@ -945,38 +960,42 @@ define <2 x float> @v_shl_v4i16(<4 x i16> %value, <4 x i16> %amount) {
define amdgpu_ps <2 x i32> @s_shl_v4i16(<4 x i16> inreg %value, <4 x i16> inreg %amount) {
; GFX6-LABEL: s_shl_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s4, s2, 16
-; GFX6-NEXT: s_lshr_b32 s5, s0, 16
-; GFX6-NEXT: s_lshr_b32 s6, s3, 16
-; GFX6-NEXT: s_lshr_b32 s7, s1, 16
-; GFX6-NEXT: s_lshl_b32 s6, s7, s6
-; GFX6-NEXT: s_lshl_b32 s1, s1, s3
-; GFX6-NEXT: s_lshl_b32 s3, s5, s4
+; GFX6-NEXT: s_lshr_b32 s4, s0, 16
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
; GFX6-NEXT: s_lshl_b32 s0, s0, s2
-; GFX6-NEXT: s_lshl_b32 s6, s6, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_lshl_b32 s2, s4, s6
+; GFX6-NEXT: s_lshr_b32 s5, s1, 16
+; GFX6-NEXT: s_lshr_b32 s7, s3, 16
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, s3
+; GFX6-NEXT: s_lshl_b32 s3, s5, s7
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s1, s1, s6
-; GFX6-NEXT: s_or_b32 s0, s0, s3
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s2
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_shl_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: s_lshl_b32 s4, s5, s4
-; GFX8-NEXT: s_lshl_b32 s1, s1, s3
-; GFX8-NEXT: s_lshl_b32 s4, s4, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s1, s4
-; GFX8-NEXT: s_lshr_b32 s3, s2, 16
; GFX8-NEXT: s_lshr_b32 s4, s0, 16
-; GFX8-NEXT: s_lshl_b32 s3, s4, s3
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
; GFX8-NEXT: s_lshl_b32 s0, s0, s2
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_lshl_b32 s2, s4, s6
+; GFX8-NEXT: s_lshr_b32 s5, s1, 16
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s1, s1, s3
+; GFX8-NEXT: s_lshl_b32 s3, s5, s7
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_shl_v4i16:
@@ -1038,51 +1057,59 @@ define <4 x float> @v_shl_v8i16(<8 x i16> %value, <8 x i16> %amount) {
; GFX6-LABEL: v_shl_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
-; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, v13, v12
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, v12, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v1, v5, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, v13, v9
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, v11, v10
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX6-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v2, v6, v2
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, v14, v10
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v14, v15, v14
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, v9, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX6-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v3, v7, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, v15, v11
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v14
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_shl_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b16_sdwa v8, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v0, v4, v0
-; GFX8-NEXT: v_lshlrev_b16_sdwa v4, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v1, v5, v1
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: v_lshlrev_b16_sdwa v4, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v2, v6, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_lshlrev_b16_sdwa v4, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_lshlrev_b16_e32 v3, v7, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v8, v4, v0
+; GFX8-NEXT: v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, v5, v1
+; GFX8-NEXT: v_lshlrev_b16_sdwa v1, v5, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, v6, v2
+; GFX8-NEXT: v_lshlrev_b16_sdwa v2, v6, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, v7, v3
+; GFX8-NEXT: v_lshlrev_b16_sdwa v3, v7, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_shl_v8i16:
@@ -1110,66 +1137,74 @@ define <4 x float> @v_shl_v8i16(<8 x i16> %value, <8 x i16> %amount) {
define amdgpu_ps <4 x i32> @s_shl_v8i16(<8 x i16> inreg %value, <8 x i16> inreg %amount) {
; GFX6-LABEL: s_shl_v8i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_lshr_b32 s8, s4, 16
-; GFX6-NEXT: s_lshr_b32 s9, s0, 16
-; GFX6-NEXT: s_lshr_b32 s10, s5, 16
-; GFX6-NEXT: s_lshr_b32 s11, s1, 16
-; GFX6-NEXT: s_lshr_b32 s12, s6, 16
-; GFX6-NEXT: s_lshr_b32 s13, s2, 16
-; GFX6-NEXT: s_lshr_b32 s14, s7, 16
-; GFX6-NEXT: s_lshr_b32 s15, s3, 16
-; GFX6-NEXT: s_lshl_b32 s14, s15, s14
-; GFX6-NEXT: s_lshl_b32 s3, s3, s7
-; GFX6-NEXT: s_lshl_b32 s7, s13, s12
-; GFX6-NEXT: s_lshl_b32 s2, s2, s6
-; GFX6-NEXT: s_lshl_b32 s6, s11, s10
-; GFX6-NEXT: s_lshl_b32 s1, s1, s5
-; GFX6-NEXT: s_lshl_b32 s5, s9, s8
+; GFX6-NEXT: s_lshr_b32 s8, s0, 16
+; GFX6-NEXT: s_lshr_b32 s12, s4, 16
; GFX6-NEXT: s_lshl_b32 s0, s0, s4
-; GFX6-NEXT: s_lshl_b32 s14, s14, 16
-; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX6-NEXT: s_lshl_b32 s7, s7, 16
-; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX6-NEXT: s_lshl_b32 s6, s6, 16
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_lshl_b32 s5, s5, 16
+; GFX6-NEXT: s_lshl_b32 s4, s8, s12
+; GFX6-NEXT: s_lshr_b32 s9, s1, 16
+; GFX6-NEXT: s_lshr_b32 s13, s5, 16
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, s5
+; GFX6-NEXT: s_lshl_b32 s5, s9, s13
; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: s_or_b32 s3, s3, s14
-; GFX6-NEXT: s_or_b32 s2, s2, s7
-; GFX6-NEXT: s_or_b32 s1, s1, s6
-; GFX6-NEXT: s_or_b32 s0, s0, s5
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_lshr_b32 s10, s2, 16
+; GFX6-NEXT: s_lshr_b32 s14, s6, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, s6
+; GFX6-NEXT: s_lshl_b32 s6, s10, s14
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_lshr_b32 s11, s3, 16
+; GFX6-NEXT: s_lshr_b32 s15, s7, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s4
+; GFX6-NEXT: s_and_b32 s4, s6, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, s7
+; GFX6-NEXT: s_lshl_b32 s7, s11, s15
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_or_b32 s3, s3, s4
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_shl_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s8, s7, 16
-; GFX8-NEXT: s_lshr_b32 s9, s3, 16
-; GFX8-NEXT: s_lshl_b32 s8, s9, s8
-; GFX8-NEXT: s_lshl_b32 s3, s3, s7
-; GFX8-NEXT: s_lshl_b32 s8, s8, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_or_b32 s3, s3, s8
-; GFX8-NEXT: s_lshr_b32 s7, s6, 16
-; GFX8-NEXT: s_lshr_b32 s8, s2, 16
-; GFX8-NEXT: s_lshl_b32 s7, s8, s7
-; GFX8-NEXT: s_lshl_b32 s2, s2, s6
-; GFX8-NEXT: s_lshl_b32 s7, s7, 16
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_or_b32 s2, s2, s7
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s1, 16
-; GFX8-NEXT: s_lshl_b32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s1, s1, s5
-; GFX8-NEXT: s_lshl_b32 s6, s6, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_or_b32 s1, s1, s6
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_lshr_b32 s6, s0, 16
-; GFX8-NEXT: s_lshl_b32 s5, s6, s5
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
; GFX8-NEXT: s_lshl_b32 s0, s0, s4
-; GFX8-NEXT: s_lshl_b32 s5, s5, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s5
+; GFX8-NEXT: s_lshl_b32 s4, s8, s12
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_lshl_b32 s1, s1, s5
+; GFX8-NEXT: s_lshl_b32 s5, s9, s13
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s5
+; GFX8-NEXT: s_lshl_b32 s2, s2, s6
+; GFX8-NEXT: s_lshl_b32 s6, s10, s14
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_lshl_b32 s3, s3, s7
+; GFX8-NEXT: s_lshl_b32 s7, s11, s15
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_shl_v8i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
index ed1b4e9cc325e..8f10f74628a50 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/smed3.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
@@ -143,13 +143,13 @@ define <2 x i16> @test_max_K0min_K1Val__v2i16(<2 x i16> %a) {
; GFX8-LABEL: test_max_K0min_K1Val__v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 17
-; GFX8-NEXT: v_min_i16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_min_i16_e32 v0, 17, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, 17
+; GFX8-NEXT: v_min_i16_e32 v1, 17, v0
+; GFX8-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mov_b32_e32 v2, -12
-; GFX8-NEXT: v_max_i16_e32 v0, -12, v0
-; GFX8-NEXT: v_max_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_i16_e32 v1, -12, v1
+; GFX8-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_max_K0min_K1Val__v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index f4d096b26bc57..83d3cde9879f4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
define i7 @v_ssubsat_i7(i7 %lhs, i7 %rhs) {
; GFX6-LABEL: v_ssubsat_i7:
@@ -588,147 +588,249 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX6-LABEL: v_ssubsat_v4i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 24, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, 24, v0
-; GFX6-NEXT: v_bfe_i32 v4, v1, 0, 8
-; GFX6-NEXT: v_bfe_i32 v5, v0, 0, 8
-; GFX6-NEXT: v_bfe_i32 v6, v1, 8, 8
-; GFX6-NEXT: v_bfe_i32 v7, v0, 8, 8
-; GFX6-NEXT: v_bfe_i32 v1, v1, 16, 8
-; GFX6-NEXT: v_bfe_i32 v0, v0, 16, 8
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT: s_movk_i32 s4, 0xff80
-; GFX6-NEXT: v_mov_b32_e32 v3, 0x7f
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT: v_max_i32_e32 v8, -1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x80000001, v8
+; GFX6-NEXT: v_min_i32_e32 v10, -1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v11, 1
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GFX6-NEXT: v_max_i32_e32 v1, v8, v1
+; GFX6-NEXT: v_min_i32_e32 v1, v1, v10
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v3
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT: v_mov_b32_e32 v9, 0x80000001
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT: v_max_i32_e32 v5, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT: v_min_i32_e32 v8, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; GFX6-NEXT: v_max_i32_e32 v2, v5, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v8
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT: v_max_i32_e32 v5, -1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT: v_min_i32_e32 v6, -1, v2
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; GFX6-NEXT: v_max_i32_e32 v3, v5, v3
+; GFX6-NEXT: v_min_i32_e32 v3, v3, v6
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT: v_max_i32_e32 v5, -1, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 24, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT: v_min_i32_e32 v6, -1, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 24, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; GFX6-NEXT: v_max_i32_e32 v4, v5, v4
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 24, v2
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v6
; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v7, v6
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v5, v4
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v3
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v3
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v2
-; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v4i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 8, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 8, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v6, 8, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v7, 8, v5
-; GFX8-NEXT: v_sub_u16_e32 v6, v7, v6
-; GFX8-NEXT: v_sub_u16_sdwa v4, sext(v5), sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_sub_u16_e32 v2, v3, v2
-; GFX8-NEXT: v_sub_u16_sdwa v0, sext(v0), sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_min_i16_e32 v6, 0x7f, v6
-; GFX8-NEXT: v_mov_b32_e32 v7, 0xffffff80
-; GFX8-NEXT: v_min_i16_e32 v4, 0x7f, v4
-; GFX8-NEXT: v_min_i16_e32 v2, 0x7f, v2
-; GFX8-NEXT: v_min_i16_e32 v0, 0x7f, v0
-; GFX8-NEXT: v_max_i16_sdwa v6, v6, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v4, 0xff80, v4
-; GFX8-NEXT: v_max_i16_sdwa v2, v2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_i16_e32 v0, 0xff80, v0
-; GFX8-NEXT: v_or_b32_sdwa v4, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, 8
+; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT: v_max_i16_e32 v9, -1, v0
+; GFX8-NEXT: v_lshrrev_b32_sdwa v6, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v8, 24, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8001, v9
+; GFX8-NEXT: v_min_i16_e32 v10, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v10, 0x8000, v10
+; GFX8-NEXT: v_max_i16_e32 v1, v9, v1
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v10
+; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
+; GFX8-NEXT: v_max_i16_e32 v1, -1, v3
+; GFX8-NEXT: v_add_u16_e32 v1, 0x8001, v1
+; GFX8-NEXT: v_min_i16_e32 v9, -1, v3
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_max_i16_e32 v1, v1, v6
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v9
+; GFX8-NEXT: v_sub_u16_e32 v1, v3, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v4
+; GFX8-NEXT: v_max_i16_e32 v6, -1, v3
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT: v_add_u16_e32 v6, 0x8001, v6
+; GFX8-NEXT: v_min_i16_e32 v7, -1, v3
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_max_i16_e32 v4, v6, v4
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v7
+; GFX8-NEXT: v_sub_u16_e32 v3, v3, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v5
+; GFX8-NEXT: v_max_i16_e32 v6, -1, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v5, 8, v8
+; GFX8-NEXT: v_add_u16_e32 v6, 0x8001, v6
+; GFX8-NEXT: v_min_i16_e32 v7, -1, v4
+; GFX8-NEXT: v_ashrrev_i16_e32 v1, 8, v1
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_max_i16_e32 v5, v6, v5
+; GFX8-NEXT: v_ashrrev_i16_e32 v0, 8, v0
+; GFX8-NEXT: v_ashrrev_i16_e32 v3, 8, v3
+; GFX8-NEXT: v_min_i16_e32 v5, v5, v7
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT: v_sub_u16_e32 v4, v4, v5
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v3
+; GFX8-NEXT: v_ashrrev_i16_e32 v4, 8, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_e32 v1, 0xff, v4
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_and_b32_e32 v2, 0xffffff00, v1
-; GFX9-NEXT: v_and_b32_e32 v3, 0xffffff00, v0
-; GFX9-NEXT: v_sub_i16 v2, v3, v2 clamp
-; GFX9-NEXT: v_lshlrev_b16_e32 v3, 8, v1
-; GFX9-NEXT: v_lshlrev_b16_e32 v4, 8, v0
-; GFX9-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-NEXT: v_sub_i16 v3, v4, v3 clamp
-; GFX9-NEXT: s_mov_b32 s5, 0xc0c0105
-; GFX9-NEXT: v_perm_b32 v2, v3, v2, s5
-; GFX9-NEXT: v_and_b32_sdwa v3, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_sdwa v4, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_sub_i16 v3, v4, v3 clamp
-; GFX9-NEXT: v_mov_b32_e32 v4, 8
-; GFX9-NEXT: v_lshlrev_b16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshlrev_b16_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_sub_i16 v0, v0, v1 clamp
-; GFX9-NEXT: v_perm_b32 v0, v0, v3, s5
-; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v2
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
+; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
+; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 24
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_ssubsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX10-NEXT: v_and_b32_e32 v4, 0xffffff00, v1
-; GFX10-NEXT: v_and_b32_e32 v5, 0xffffff00, v0
-; GFX10-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX10-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX10-NEXT: v_and_b32_e32 v6, 0xffffff00, v3
-; GFX10-NEXT: v_and_b32_e32 v7, 0xffffff00, v2
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_sub_nc_i16 v4, v5, v4 clamp
-; GFX10-NEXT: v_sub_nc_i16 v0, v0, v1 clamp
-; GFX10-NEXT: v_sub_nc_i16 v1, v7, v6 clamp
-; GFX10-NEXT: v_sub_nc_i16 v2, v2, v3 clamp
-; GFX10-NEXT: v_perm_b32 v0, v0, v4, 0xc0c0105
-; GFX10-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX10-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
+; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-NEXT: v_pk_ashrrev_i16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_ssubsat_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff00, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v2.h, 0xff00, v0.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v3.l, 0xff00, v1.h
-; GFX11-TRUE16-NEXT: v_and_b16 v3.h, 0xff00, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v2.l, v2.h, v2.l clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v0.l, v0.l, v1.l clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v1.l, v3.h, v3.l clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v3.l, v0.h, v1.h clamp
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v2, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v6 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_sub_i16 v1, v2, v1 clamp
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_pk_ashrrev_i16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_ssubsat_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffffff00, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffffff00, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v6, 0xffffff00, v3
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffffff00, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v4, v5, v4 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v0, v0, v1 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v1, v7, v6 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v2, v2, v3 clamp
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v0, v4, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v2, v1, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v3, 16, v5
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v6, v0, 16
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
+; GFX11-FAKE16-NEXT: v_pk_sub_i16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_ashrrev_i16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_ashrrev_i16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xff, v1, v2
+; GFX11-FAKE16-NEXT: v_or3_b32 v0, v1, v3, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -740,171 +842,366 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
define amdgpu_ps i32 @s_ssubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
; GFX6-LABEL: s_ssubsat_v4i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s2, s1, 24
-; GFX6-NEXT: s_ashr_i32 s3, s0, 24
-; GFX6-NEXT: s_sext_i32_i8 s4, s1
-; GFX6-NEXT: s_sext_i32_i8 s5, s0
-; GFX6-NEXT: s_bfe_i32 s6, s1, 0x80008
-; GFX6-NEXT: s_bfe_i32 s7, s0, 0x80008
-; GFX6-NEXT: s_bfe_i32 s1, s1, 0x80010
-; GFX6-NEXT: s_bfe_i32 s0, s0, 0x80010
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7f
+; GFX6-NEXT: s_lshr_b32 s2, s0, 8
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_lshr_b32 s4, s0, 24
+; GFX6-NEXT: s_lshl_b32 s0, s0, 24
+; GFX6-NEXT: s_max_i32 s8, s0, -1
+; GFX6-NEXT: s_lshr_b32 s5, s1, 8
+; GFX6-NEXT: s_lshr_b32 s6, s1, 16
+; GFX6-NEXT: s_lshr_b32 s7, s1, 24
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000001
+; GFX6-NEXT: s_min_i32 s9, s0, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000000
+; GFX6-NEXT: s_max_i32 s1, s8, s1
+; GFX6-NEXT: s_min_i32 s1, s1, s9
; GFX6-NEXT: s_sub_i32 s0, s0, s1
-; GFX6-NEXT: s_sub_i32 s2, s3, s2
-; GFX6-NEXT: v_med3_i32 v3, s0, v0, v1
-; GFX6-NEXT: v_med3_i32 v2, s2, v0, v1
-; GFX6-NEXT: v_and_b32_e32 v3, 0xff, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: s_sub_i32 s0, s7, s6
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX6-NEXT: v_med3_i32 v3, s0, v0, v1
-; GFX6-NEXT: s_sub_i32 s0, s5, s4
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 24
+; GFX6-NEXT: s_lshl_b32 s2, s5, 24
+; GFX6-NEXT: s_max_i32 s5, s1, -1
+; GFX6-NEXT: s_add_i32 s5, s5, 0x80000001
+; GFX6-NEXT: s_min_i32 s8, s1, -1
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000000
+; GFX6-NEXT: s_max_i32 s2, s5, s2
+; GFX6-NEXT: s_min_i32 s2, s2, s8
+; GFX6-NEXT: s_sub_i32 s1, s1, s2
+; GFX6-NEXT: s_lshl_b32 s2, s3, 24
+; GFX6-NEXT: s_max_i32 s5, s2, -1
+; GFX6-NEXT: s_lshl_b32 s3, s6, 24
+; GFX6-NEXT: s_add_i32 s5, s5, 0x80000001
+; GFX6-NEXT: s_min_i32 s6, s2, -1
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000000
+; GFX6-NEXT: s_max_i32 s3, s5, s3
+; GFX6-NEXT: s_min_i32 s3, s3, s6
+; GFX6-NEXT: s_sub_i32 s2, s2, s3
+; GFX6-NEXT: s_lshl_b32 s3, s4, 24
+; GFX6-NEXT: s_max_i32 s5, s3, -1
+; GFX6-NEXT: s_ashr_i32 s1, s1, 24
+; GFX6-NEXT: s_lshl_b32 s4, s7, 24
+; GFX6-NEXT: s_add_i32 s5, s5, 0x80000001
+; GFX6-NEXT: s_min_i32 s6, s3, -1
+; GFX6-NEXT: s_ashr_i32 s0, s0, 24
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000000
+; GFX6-NEXT: s_max_i32 s4, s5, s4
+; GFX6-NEXT: s_and_b32 s1, s1, 0xff
+; GFX6-NEXT: s_ashr_i32 s2, s2, 24
+; GFX6-NEXT: s_min_i32 s4, s4, s6
+; GFX6-NEXT: s_and_b32 s0, s0, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 8
+; GFX6-NEXT: s_sub_i32 s3, s3, s4
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s2, 0xff
+; GFX6-NEXT: s_ashr_i32 s3, s3, 24
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_and_b32 s1, s3, 0xff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ssubsat_v4i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s0, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: s_ashr_i32 s6, s1, 24
-; GFX8-NEXT: s_ashr_i32 s7, s0, 24
-; GFX8-NEXT: s_sext_i32_i16 s2, s1
-; GFX8-NEXT: s_sext_i32_i16 s3, s0
-; GFX8-NEXT: s_bfe_i32 s4, s4, 0x80000
-; GFX8-NEXT: s_bfe_i32 s5, s5, 0x80000
-; GFX8-NEXT: s_sub_i32 s6, s7, s6
-; GFX8-NEXT: s_ashr_i32 s2, s2, 8
-; GFX8-NEXT: s_ashr_i32 s3, s3, 8
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: s_lshr_b32 s4, s0, 24
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s8, s0
+; GFX8-NEXT: s_max_i32 s9, s8, -1
+; GFX8-NEXT: s_lshr_b32 s5, s1, 8
+; GFX8-NEXT: s_lshr_b32 s6, s1, 16
+; GFX8-NEXT: s_lshr_b32 s7, s1, 24
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_add_i32 s9, s9, 0x8001
+; GFX8-NEXT: s_min_i32 s8, s8, -1
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8000
+; GFX8-NEXT: s_max_i32 s1, s9, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_min_i32 s1, s1, s8
+; GFX8-NEXT: s_sub_i32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s5, 8
+; GFX8-NEXT: s_sext_i32_i16 s5, s1
+; GFX8-NEXT: s_max_i32 s8, s5, -1
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8001
+; GFX8-NEXT: s_min_i32 s5, s5, -1
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8000
+; GFX8-NEXT: s_max_i32 s2, s8, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_min_i32 s2, s2, s5
+; GFX8-NEXT: s_sub_i32 s1, s1, s2
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: s_sext_i32_i16 s5, s2
+; GFX8-NEXT: s_lshl_b32 s3, s6, 8
+; GFX8-NEXT: s_max_i32 s6, s5, -1
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8001
+; GFX8-NEXT: s_min_i32 s5, s5, -1
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8000
+; GFX8-NEXT: s_max_i32 s3, s6, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_min_i32 s3, s3, s5
+; GFX8-NEXT: s_sub_i32 s2, s2, s3
+; GFX8-NEXT: s_lshl_b32 s3, s4, 8
+; GFX8-NEXT: s_sext_i32_i16 s5, s3
+; GFX8-NEXT: s_max_i32 s6, s5, -1
+; GFX8-NEXT: s_lshl_b32 s4, s7, 8
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8001
+; GFX8-NEXT: s_min_i32 s5, s5, -1
; GFX8-NEXT: s_sext_i32_i16 s6, s6
-; GFX8-NEXT: v_mov_b32_e32 v0, 0xffffff80
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7f
-; GFX8-NEXT: s_sub_i32 s4, s4, s5
-; GFX8-NEXT: s_bfe_i32 s0, s0, 0x80000
-; GFX8-NEXT: s_bfe_i32 s1, s1, 0x80000
-; GFX8-NEXT: v_med3_i32 v2, s6, v0, v1
; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sub_i32 s2, s3, s2
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX8-NEXT: v_med3_i32 v3, s4, v0, v1
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_med3_i32 v3, s2, v0, v1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8000
+; GFX8-NEXT: s_max_i32 s4, s6, s4
; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX8-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_ashr_i32 s1, s1, 8
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_ashr_i32 s0, s0, 8
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_min_i32 s4, s4, s5
+; GFX8-NEXT: s_and_b32 s1, s1, 0xff
+; GFX8-NEXT: s_ashr_i32 s2, s2, 8
+; GFX8-NEXT: s_sub_i32 s3, s3, s4
+; GFX8-NEXT: s_and_b32 s0, s0, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s2, 0xff
+; GFX8-NEXT: s_ashr_i32 s3, s3, 8
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_and_b32 s1, s3, 0xff
+; GFX8-NEXT: s_lshl_b32 s1, s1, 24
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_v4i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s3, s1, 16
-; GFX9-NEXT: s_lshr_b32 s5, s1, 24
-; GFX9-NEXT: s_and_b32 s6, s1, 0xff00
-; GFX9-NEXT: s_lshl_b32 s1, s1, 8
-; GFX9-NEXT: s_lshr_b32 s2, s0, 16
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
; GFX9-NEXT: s_lshr_b32 s4, s0, 24
-; GFX9-NEXT: s_and_b32 s7, s0, 0xff00
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: s_lshl_b32 s0, s0, 8
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_lshl_b32 s5, s5, 8
-; GFX9-NEXT: v_sub_i16 v0, s7, v0 clamp
-; GFX9-NEXT: v_sub_i16 v1, s0, v1 clamp
-; GFX9-NEXT: s_mov_b32 s0, 0xc0c0105
-; GFX9-NEXT: s_lshl_b32 s1, s3, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
; GFX9-NEXT: s_lshl_b32 s4, s4, 8
-; GFX9-NEXT: v_perm_b32 v0, v1, v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_lshl_b32 s2, s2, 8
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
-; GFX9-NEXT: v_sub_i16 v1, s4, v1 clamp
-; GFX9-NEXT: v_sub_i16 v2, s2, v2 clamp
-; GFX9-NEXT: v_perm_b32 v1, v2, v1, s0
-; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX9-NEXT: s_lshr_b32 s5, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshr_b32 s6, s1, 16
+; GFX9-NEXT: s_lshr_b32 s7, s1, 24
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s6, s7
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT: v_pk_sub_i16 v0, s0, v0 clamp
; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_pk_sub_i16 v0, s2, v0 clamp
+; GFX9-NEXT: s_sext_i32_i16 s2, s0
+; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_sext_i32_i16 s2, s1
+; GFX9-NEXT: s_ashr_i32 s1, s1, 16
+; GFX9-NEXT: s_ashr_i32 s2, s2, s3
+; GFX9-NEXT: s_ashr_i32 s1, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT: s_and_b32 s2, s0, 0xff
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX9-NEXT: s_or_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s1, s1, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_ssubsat_v4i8:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_lshr_b32 s2, s0, 16
-; GFX10-NEXT: s_lshr_b32 s3, s1, 16
+; GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10-NEXT: s_lshr_b32 s5, s1, 8
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
; GFX10-NEXT: s_lshr_b32 s4, s0, 24
-; GFX10-NEXT: s_lshr_b32 s5, s1, 24
-; GFX10-NEXT: s_lshl_b32 s4, s4, 8
+; GFX10-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10-NEXT: s_lshr_b32 s7, s1, 24
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s4, s6, s7
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x80008
; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s3, s2, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_lshr_b32 s5, s4, 16
+; GFX10-NEXT: s_lshl_b32 s2, s2, 0x80008
; GFX10-NEXT: s_lshl_b32 s3, s3, 8
-; GFX10-NEXT: s_lshl_b32 s2, s2, 8
-; GFX10-NEXT: v_sub_nc_i16 v0, s4, s5 clamp
-; GFX10-NEXT: v_sub_nc_i16 v1, s2, s3 clamp
-; GFX10-NEXT: s_and_b32 s6, s1, 0xff00
-; GFX10-NEXT: s_and_b32 s7, s0, 0xff00
-; GFX10-NEXT: s_lshl_b32 s1, s1, 8
-; GFX10-NEXT: s_lshl_b32 s0, s0, 8
-; GFX10-NEXT: v_sub_nc_i16 v2, s7, s6 clamp
-; GFX10-NEXT: v_sub_nc_i16 v3, s0, s1 clamp
-; GFX10-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX10-NEXT: v_perm_b32 v1, v3, v2, 0xc0c0105
-; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT: s_lshl_b32 s4, s4, 0x80008
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: v_pk_sub_i16 v0, s0, s1 clamp
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s4, s5
+; GFX10-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX10-NEXT: v_pk_sub_i16 v1, s0, s1 clamp
; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_sext_i32_i16 s2, s0
+; GFX10-NEXT: s_ashr_i32 s0, s0, 16
+; GFX10-NEXT: s_ashr_i32 s2, s2, s3
+; GFX10-NEXT: s_ashr_i32 s0, s0, 8
+; GFX10-NEXT: s_sext_i32_i16 s4, s1
+; GFX10-NEXT: s_ashr_i32 s1, s1, 16
+; GFX10-NEXT: s_ashr_i32 s3, s4, s3
+; GFX10-NEXT: s_ashr_i32 s1, s1, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_and_b32 s3, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, 16
+; GFX10-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s1
; GFX10-NEXT: ; return to shader part epilog
;
; GFX11-TRUE16-LABEL: s_ssubsat_v4i8:
; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, s2, s3 clamp
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX11-TRUE16-NEXT: v_pk_sub_i16 v1, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s2, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_ashr_i32 s2, s2, s3
+; GFX11-TRUE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_sext_i32_i16 s4, s1
+; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_ashr_i32 s3, s4, s3
+; GFX11-TRUE16-NEXT: s_ashr_i32 s1, s1, 8
; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s7, s2, 0xff00
-; GFX11-TRUE16-NEXT: s_and_b32 s6, s3, 0xff00
-; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v0.l, s7, s6 clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v1.l, s2, s3 clamp
-; GFX11-TRUE16-NEXT: s_and_b32 s4, s1, 0xff00
-; GFX11-TRUE16-NEXT: s_and_b32 s5, s0, 0xff00
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 8
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v2.l, s5, s4 clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_i16 v3.l, s0, s1 clamp
-; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0xc0c0105
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 24
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-TRUE16-NEXT: ; return to shader part epilog
;
; GFX11-FAKE16-LABEL: s_ssubsat_v4i8:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 16
-; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s1, 16
-; GFX11-FAKE16-NEXT: s_and_b32 s7, s2, 0xff00
-; GFX11-FAKE16-NEXT: s_and_b32 s6, s3, 0xff00
-; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v0, s7, s6 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v1, s2, s3 clamp
-; GFX11-FAKE16-NEXT: s_and_b32 s4, s1, 0xff00
-; GFX11-FAKE16-NEXT: s_and_b32 s5, s0, 0xff00
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 8
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 8
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v2, s5, s4 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_i16 v3, s0, s1 clamp
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v3, v2, 0xc0c0105
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: v_pk_sub_i16 v0, s2, s3 clamp
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s3, 0x80008
+; GFX11-FAKE16-NEXT: v_pk_sub_i16 v1, s0, s1 clamp
; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s2, s0
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_ashr_i32 s2, s2, s3
+; GFX11-FAKE16-NEXT: s_ashr_i32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_sext_i32_i16 s4, s1
+; GFX11-FAKE16-NEXT: s_ashr_i32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_ashr_i32 s3, s4, s3
+; GFX11-FAKE16-NEXT: s_ashr_i32 s1, s1, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s3, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 24
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -2710,42 +3007,55 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: v_ssubsat_v2i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v0
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v3, v2
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v4, -1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x80000001, v4
+; GFX6-NEXT: v_min_i32_e32 v6, -1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v7, 1
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v7
+; GFX6-NEXT: v_max_i32_e32 v1, v4, v1
+; GFX6-NEXT: v_min_i32_e32 v1, v1, v6
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v3
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v3
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_mov_b32_e32 v5, 0x80000001
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_max_i32_e32 v3, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5
+; GFX6-NEXT: v_min_i32_e32 v4, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 0x80000000, v4
+; GFX6-NEXT: v_max_i32_e32 v2, v3, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v4
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v3, v2
-; GFX8-NEXT: v_sub_u16_e32 v2, v3, v2
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v3, 0xffff8000, v3
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v0, v1
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v1, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v1, 0xffff8000, v1
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_e32 v2, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v2, 0x8001, v2
+; GFX8-NEXT: v_min_i16_e32 v3, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT: v_max_i16_e32 v2, v2, v1
+; GFX8-NEXT: v_min_i16_e32 v2, v2, v3
+; GFX8-NEXT: v_mov_b32_e32 v3, -1
+; GFX8-NEXT: v_max_i16_sdwa v4, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v4, 0x8001, v4
+; GFX8-NEXT: v_min_i16_sdwa v3, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v3, 0x8000, v3
+; GFX8-NEXT: v_max_i16_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v3
+; GFX8-NEXT: v_sub_u16_e32 v2, v0, v2
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v2i16:
@@ -2766,57 +3076,66 @@ define <2 x i16> @v_ssubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: s_ssubsat_v2i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s2, s1, 16
-; GFX6-NEXT: s_ashr_i32 s3, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: s_sub_i32 s2, s3, s2
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX6-NEXT: s_lshr_b32 s2, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s4, s0, -1
+; GFX6-NEXT: s_lshr_b32 s3, s1, 16
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_add_i32 s4, s4, 0x80000001
+; GFX6-NEXT: s_min_i32 s5, s0, -1
+; GFX6-NEXT: s_add_i32 s5, s5, 0x80000000
+; GFX6-NEXT: s_max_i32 s1, s4, s1
+; GFX6-NEXT: s_min_i32 s1, s1, s5
; GFX6-NEXT: s_sub_i32 s0, s0, s1
-; GFX6-NEXT: v_med3_i32 v2, s2, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: s_lshl_b32 s2, s3, 16
+; GFX6-NEXT: s_max_i32 s3, s1, -1
+; GFX6-NEXT: s_add_i32 s3, s3, 0x80000001
+; GFX6-NEXT: s_min_i32 s4, s1, -1
+; GFX6-NEXT: s_add_i32 s4, s4, 0x80000000
+; GFX6-NEXT: s_max_i32 s2, s3, s2
+; GFX6-NEXT: s_min_i32 s2, s2, s4
+; GFX6-NEXT: s_sub_i32 s1, s1, s2
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ssubsat_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_sub_i32 s6, s3, s2
-; GFX8-NEXT: s_lshl_b32 s4, s6, 16
-; GFX8-NEXT: s_ashr_i32 s2, s1, 16
-; GFX8-NEXT: s_ashr_i32 s3, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s3, s2
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s4, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT: s_sext_i32_i16 s4, s6
-; GFX8-NEXT: s_lshr_b32 s4, s4, 15
-; GFX8-NEXT: s_xor_b32 s4, s4, 0x8000
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX8-NEXT: s_cselect_b32 s2, s4, s6
-; GFX8-NEXT: s_sub_i32 s5, s0, s1
-; GFX8-NEXT: s_lshl_b32 s4, s2, 16
-; GFX8-NEXT: s_lshl_b32 s2, s5, 16
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s0, s1
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s2, 0
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: s_sext_i32_i16 s2, s5
-; GFX8-NEXT: s_lshr_b32 s2, s2, 15
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX8-NEXT: s_cselect_b32 s0, s2, s5
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s0
+; GFX8-NEXT: s_max_i32 s5, s4, -1
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8001
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: s_min_i32 s4, s4, -1
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8000
+; GFX8-NEXT: s_max_i32 s1, s5, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_min_i32 s1, s1, s4
+; GFX8-NEXT: s_sub_i32 s0, s0, s1
+; GFX8-NEXT: s_sext_i32_i16 s1, s2
+; GFX8-NEXT: s_max_i32 s4, s1, -1
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8001
+; GFX8-NEXT: s_min_i32 s1, s1, -1
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_add_i32 s1, s1, 0x8000
+; GFX8-NEXT: s_max_i32 s3, s4, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s1, s1
+; GFX8-NEXT: s_min_i32 s1, s3, s1
+; GFX8-NEXT: s_sub_i32 s1, s2, s1
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_v2i16:
@@ -2839,41 +3158,56 @@ define amdgpu_ps i32 @s_ssubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
; GFX6-LABEL: ssubsat_v2i16_sv:
; GFX6: ; %bb.0:
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX6-NEXT: s_ashr_i32 s1, s0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s1, v1
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s2, s0, -1
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: s_add_i32 s2, s2, 0x80000001
+; GFX6-NEXT: s_min_i32 s3, s0, -1
+; GFX6-NEXT: s_add_i32 s3, s3, 0x80000000
+; GFX6-NEXT: v_max_i32_e32 v0, s2, v0
+; GFX6-NEXT: v_min_i32_e32 v0, s3, v0
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_med3_i32 v1, v1, s1, v2
-; GFX6-NEXT: v_med3_i32 v0, v0, s1, v2
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
+; GFX6-NEXT: s_max_i32 s1, s0, -1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: s_add_i32 s1, s1, 0x80000001
+; GFX6-NEXT: s_min_i32 s2, s0, -1
+; GFX6-NEXT: s_add_i32 s2, s2, 0x80000000
+; GFX6-NEXT: v_max_i32_e32 v1, s1, v1
+; GFX6-NEXT: v_min_i32_e32 v1, s2, v1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s0, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_v2i16_sv:
; GFX8: ; %bb.0:
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX8-NEXT: s_sext_i32_i16 s2, s0
+; GFX8-NEXT: s_max_i32 s3, s2, -1
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8001
+; GFX8-NEXT: s_min_i32 s2, s2, -1
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, s1, v1
-; GFX8-NEXT: v_sub_u16_e32 v1, s1, v1
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[2:3], 0, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, s0, v0
-; GFX8-NEXT: v_sub_u16_e32 v0, s0, v0
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8000
+; GFX8-NEXT: v_max_i16_e32 v1, s3, v0
+; GFX8-NEXT: v_min_i16_e32 v1, s2, v1
+; GFX8-NEXT: s_sext_i32_i16 s2, s1
+; GFX8-NEXT: s_max_i32 s3, s2, -1
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8001
+; GFX8-NEXT: s_min_i32 s2, s2, -1
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8000
+; GFX8-NEXT: v_max_i16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v0, s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_sub_u16_e32 v1, s0, v1
+; GFX8-NEXT: v_sub_u16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_v2i16_sv:
@@ -2893,41 +3227,55 @@ define amdgpu_ps float @ssubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
define amdgpu_ps float @ssubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
; GFX6-LABEL: ssubsat_v2i16_vs:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s1, s0, 16
-; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v0
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s1, v1
-; GFX6-NEXT: s_movk_i32 s1, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: v_med3_i32 v1, v1, s1, v2
-; GFX6-NEXT: v_med3_i32 v0, v0, s1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v2, -1, v0
+; GFX6-NEXT: s_lshr_b32 s1, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 0x80000001, v2
+; GFX6-NEXT: v_min_i32_e32 v4, -1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v5, 1
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v5
+; GFX6-NEXT: v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_mov_b32_e32 v3, 0x80000001
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
+; GFX6-NEXT: v_max_i32_e32 v2, -1, v1
+; GFX6-NEXT: s_lshl_b32 s0, s1, 16
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_min_i32_e32 v3, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 0x80000000, v3
+; GFX6-NEXT: v_max_i32_e32 v2, s0, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v3
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: ssubsat_v2i16_vs:
; GFX8: ; %bb.0:
+; GFX8-NEXT: v_max_i16_e32 v1, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v1, 0x8001, v1
+; GFX8-NEXT: v_min_i16_e32 v2, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT: v_max_i16_e32 v1, s0, v1
+; GFX8-NEXT: v_min_i16_e32 v1, v1, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, -1
+; GFX8-NEXT: v_max_i16_sdwa v3, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, s1, v1
-; GFX8-NEXT: v_subrev_u16_e32 v1, s1, v1
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[2:3], 0, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[2:3]
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX8-NEXT: v_cmp_gt_i16_e32 vcc, s0, v0
-; GFX8-NEXT: v_subrev_u16_e32 v0, s0, v0
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[0:1], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[0:1]
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v3, 0x8001, v3
+; GFX8-NEXT: v_min_i16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v2, 0x8000, v2
+; GFX8-NEXT: v_max_i16_e32 v3, s1, v3
+; GFX8-NEXT: v_min_i16_e32 v2, v3, v2
+; GFX8-NEXT: v_sub_u16_e32 v1, v0, v1
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: ssubsat_v2i16_vs:
@@ -2959,71 +3307,96 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
; GFX6-LABEL: v_ssubsat_v4i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v2
-; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v0
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v7, v6
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v8, -1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, 0x80000001, v8
+; GFX6-NEXT: v_min_i32_e32 v10, -1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v11, 1
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, v10, v11
+; GFX6-NEXT: v_max_i32_e32 v2, v8, v2
+; GFX6-NEXT: v_min_i32_e32 v2, v2, v10
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v5, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-NEXT: v_mov_b32_e32 v9, 0x80000001
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
+; GFX6-NEXT: v_max_i32_e32 v6, -1, v2
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v9
+; GFX6-NEXT: v_min_i32_e32 v8, -1, v2
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v11
+; GFX6-NEXT: v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v4
+; GFX6-NEXT: v_max_i32_e32 v4, -1, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, v4, v9
+; GFX6-NEXT: v_min_i32_e32 v6, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; GFX6-NEXT: v_max_i32_e32 v3, v4, v3
+; GFX6-NEXT: v_min_i32_e32 v3, v3, v6
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v3
-; GFX6-NEXT: v_med3_i32 v6, v6, s4, v7
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v7
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v7
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v7
-; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_max_i32_e32 v5, -1, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v9
+; GFX6-NEXT: v_min_i32_e32 v6, -1, v3
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v11
+; GFX6-NEXT: v_max_i32_e32 v4, v5, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v6
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v3
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v3
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v5, v4
-; GFX8-NEXT: v_sub_u16_e32 v4, v5, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v5, 0xffff8000, v5
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v0, v2
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v2, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v2, 0xffff8000, v2
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v4, v2
-; GFX8-NEXT: v_sub_u16_e32 v2, v4, v2
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v1, v3
-; GFX8-NEXT: v_sub_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v3, 0xffff8000, v3
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_e32 v4, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v4, 0x8001, v4
+; GFX8-NEXT: v_min_i16_e32 v5, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v5, 0x8000, v5
+; GFX8-NEXT: v_max_i16_e32 v4, v4, v2
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v5
+; GFX8-NEXT: v_mov_b32_e32 v5, -1
+; GFX8-NEXT: v_max_i16_sdwa v6, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v6, 0x8001, v6
+; GFX8-NEXT: v_min_i16_sdwa v7, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_max_i16_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v6, -1, v1
+; GFX8-NEXT: v_min_i16_e32 v2, v2, v7
+; GFX8-NEXT: v_add_u16_e32 v6, 0x8001, v6
+; GFX8-NEXT: v_min_i16_e32 v7, -1, v1
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_max_i16_e32 v6, v6, v3
+; GFX8-NEXT: v_min_i16_e32 v6, v6, v7
+; GFX8-NEXT: v_max_i16_sdwa v7, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8001, v7
+; GFX8-NEXT: v_min_i16_sdwa v5, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v5, 0x8000, v5
+; GFX8-NEXT: v_max_i16_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v3, v3, v5
+; GFX8-NEXT: v_sub_u16_e32 v4, v0, v4
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v2, v1, v6
+; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v4i16:
@@ -3047,102 +3420,122 @@ define <2 x float> @v_ssubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
define amdgpu_ps <2 x i32> @s_ssubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inreg %rhs) {
; GFX6-LABEL: s_ssubsat_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s6, s3, 16
-; GFX6-NEXT: s_ashr_i32 s7, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: s_sub_i32 s6, s7, s6
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: s_sub_i32 s1, s1, s3
-; GFX6-NEXT: s_ashr_i32 s4, s2, 16
-; GFX6-NEXT: s_ashr_i32 s5, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_med3_i32 v2, s6, v0, v1
-; GFX6-NEXT: v_med3_i32 v3, s1, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: s_sub_i32 s1, s5, s4
+; GFX6-NEXT: s_lshr_b32 s4, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s8, s0, -1
+; GFX6-NEXT: s_lshr_b32 s6, s2, 16
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000001
+; GFX6-NEXT: s_min_i32 s9, s0, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000000
+; GFX6-NEXT: s_max_i32 s2, s8, s2
+; GFX6-NEXT: s_min_i32 s2, s2, s9
; GFX6-NEXT: s_sub_i32 s0, s0, s2
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_med3_i32 v3, s1, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s1, v2
+; GFX6-NEXT: s_lshl_b32 s2, s4, 16
+; GFX6-NEXT: s_lshl_b32 s4, s6, 16
+; GFX6-NEXT: s_max_i32 s6, s2, -1
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000001
+; GFX6-NEXT: s_min_i32 s8, s2, -1
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000000
+; GFX6-NEXT: s_max_i32 s4, s6, s4
+; GFX6-NEXT: s_lshr_b32 s5, s1, 16
+; GFX6-NEXT: s_min_i32 s4, s4, s8
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_sub_i32 s2, s2, s4
+; GFX6-NEXT: s_max_i32 s4, s1, -1
+; GFX6-NEXT: s_lshr_b32 s7, s3, 16
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_add_i32 s4, s4, 0x80000001
+; GFX6-NEXT: s_min_i32 s6, s1, -1
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000000
+; GFX6-NEXT: s_max_i32 s3, s4, s3
+; GFX6-NEXT: s_min_i32 s3, s3, s6
+; GFX6-NEXT: s_sub_i32 s1, s1, s3
+; GFX6-NEXT: s_lshl_b32 s3, s5, 16
+; GFX6-NEXT: s_max_i32 s5, s3, -1
+; GFX6-NEXT: s_lshl_b32 s4, s7, 16
+; GFX6-NEXT: s_add_i32 s5, s5, 0x80000001
+; GFX6-NEXT: s_min_i32 s6, s3, -1
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000000
+; GFX6-NEXT: s_max_i32 s4, s5, s4
+; GFX6-NEXT: s_ashr_i32 s2, s2, 16
+; GFX6-NEXT: s_min_i32 s4, s4, s6
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_sub_i32 s3, s3, s4
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_ashr_i32 s3, s3, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s2
+; GFX6-NEXT: s_and_b32 s2, s3, 0xffff
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s2
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ssubsat_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: s_sub_i32 s8, s5, s4
-; GFX8-NEXT: s_lshl_b32 s6, s8, 16
-; GFX8-NEXT: s_ashr_i32 s4, s3, 16
-; GFX8-NEXT: s_ashr_i32 s5, s1, 16
-; GFX8-NEXT: s_cmp_lt_i32 s5, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_sext_i32_i16 s6, s8
-; GFX8-NEXT: s_lshr_b32 s6, s6, 15
-; GFX8-NEXT: s_xor_b32 s6, s6, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s4, s6, s8
-; GFX8-NEXT: s_sub_i32 s9, s1, s3
-; GFX8-NEXT: s_lshl_b32 s8, s4, 16
-; GFX8-NEXT: s_lshl_b32 s6, s9, 16
-; GFX8-NEXT: s_sext_i32_i16 s3, s3
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_cmp_lt_i32 s1, s3
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_sext_i32_i16 s1, s9
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8-NEXT: s_lshr_b32 s3, s2, 16
+; GFX8-NEXT: s_sext_i32_i16 s8, s0
+; GFX8-NEXT: s_max_i32 s9, s8, -1
+; GFX8-NEXT: s_add_i32 s9, s9, 0x8001
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
+; GFX8-NEXT: s_min_i32 s8, s8, -1
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8000
+; GFX8-NEXT: s_max_i32 s2, s9, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s2
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
; GFX8-NEXT: s_lshr_b32 s4, s0, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_sub_i32 s3, s4, s3
-; GFX8-NEXT: s_or_b32 s1, s1, s8
-; GFX8-NEXT: s_lshl_b32 s6, s3, 16
-; GFX8-NEXT: s_ashr_i32 s4, s2, 16
-; GFX8-NEXT: s_ashr_i32 s5, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s5, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_sext_i32_i16 s6, s3
-; GFX8-NEXT: s_lshr_b32 s6, s6, 15
-; GFX8-NEXT: s_xor_b32 s6, s6, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s3, s6, s3
-; GFX8-NEXT: s_sub_i32 s7, s0, s2
-; GFX8-NEXT: s_lshl_b32 s6, s3, 16
-; GFX8-NEXT: s_lshl_b32 s4, s7, 16
+; GFX8-NEXT: s_min_i32 s2, s2, s8
+; GFX8-NEXT: s_sub_i32 s0, s0, s2
+; GFX8-NEXT: s_sext_i32_i16 s2, s4
+; GFX8-NEXT: s_max_i32 s8, s2, -1
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8001
+; GFX8-NEXT: s_min_i32 s2, s2, -1
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_add_i32 s2, s2, 0x8000
+; GFX8-NEXT: s_max_i32 s6, s8, s6
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s0, s2
-; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s4, 0
-; GFX8-NEXT: s_sext_i32_i16 s0, s7
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 15
-; GFX8-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5]
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x8000
-; GFX8-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX8-NEXT: s_cselect_b32 s0, s0, s7
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s6
+; GFX8-NEXT: s_min_i32 s2, s6, s2
+; GFX8-NEXT: s_sub_i32 s2, s4, s2
+; GFX8-NEXT: s_sext_i32_i16 s4, s1
+; GFX8-NEXT: s_max_i32 s6, s4, -1
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8001
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
+; GFX8-NEXT: s_min_i32 s4, s4, -1
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8000
+; GFX8-NEXT: s_max_i32 s3, s6, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_lshr_b32 s5, s1, 16
+; GFX8-NEXT: s_min_i32 s3, s3, s4
+; GFX8-NEXT: s_sub_i32 s1, s1, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s5
+; GFX8-NEXT: s_max_i32 s4, s3, -1
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8001
+; GFX8-NEXT: s_min_i32 s3, s3, -1
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s7
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8000
+; GFX8-NEXT: s_max_i32 s4, s4, s6
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_min_i32 s3, s4, s3
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_sub_i32 s3, s5, s3
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_v4i16:
@@ -3182,100 +3575,137 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
; GFX6-LABEL: v_ssubsat_v6i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v8, 16, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v10, 16, v3
-; GFX6-NEXT: v_ashrrev_i32_e32 v11, 16, v0
-; GFX6-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_sub_i32_e32 v10, vcc, v11, v10
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v11, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v12, -1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v12, vcc, 0x80000001, v12
+; GFX6-NEXT: v_min_i32_e32 v14, -1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v15, 1
+; GFX6-NEXT: v_add_i32_e32 v14, vcc, v14, v15
+; GFX6-NEXT: v_max_i32_e32 v3, v12, v3
+; GFX6-NEXT: v_min_i32_e32 v3, v3, v14
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v3
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v9, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX6-NEXT: v_mov_b32_e32 v13, 0x80000001
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v9
+; GFX6-NEXT: v_max_i32_e32 v9, -1, v3
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v13
+; GFX6-NEXT: v_min_i32_e32 v12, -1, v3
+; GFX6-NEXT: v_add_i32_e32 v12, vcc, v12, v15
+; GFX6-NEXT: v_max_i32_e32 v6, v9, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v12
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v6
+; GFX6-NEXT: v_max_i32_e32 v6, -1, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT: v_min_i32_e32 v9, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v15
+; GFX6-NEXT: v_max_i32_e32 v4, v6, v4
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v9
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v4
-; GFX6-NEXT: v_med3_i32 v3, v3, s4, v11
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v11
-; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v2
-; GFX6-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v7, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX6-NEXT: v_max_i32_e32 v7, -1, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, v7, v13
+; GFX6-NEXT: v_min_i32_e32 v9, -1, v4
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v15
+; GFX6-NEXT: v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v9
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v6
+; GFX6-NEXT: v_max_i32_e32 v6, -1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, v6, v13
+; GFX6-NEXT: v_min_i32_e32 v7, -1, v2
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, v7, v15
+; GFX6-NEXT: v_max_i32_e32 v5, v6, v5
+; GFX6-NEXT: v_min_i32_e32 v5, v5, v7
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v5
-; GFX6-NEXT: v_med3_i32 v10, v10, s4, v11
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v11
-; GFX6-NEXT: v_med3_i32 v3, v3, s4, v11
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v11
-; GFX6-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v8
+; GFX6-NEXT: v_max_i32_e32 v7, -1, v5
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v11
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, v7, v13
+; GFX6-NEXT: v_min_i32_e32 v8, -1, v5
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v15
+; GFX6-NEXT: v_max_i32_e32 v6, v7, v6
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v8
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v5, v6
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v5
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v6i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v7, v6
-; GFX8-NEXT: v_sub_u16_e32 v6, v7, v6
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v6
-; GFX8-NEXT: v_ashrrev_i16_e32 v7, 15, v6
-; GFX8-NEXT: v_xor_b32_e32 v7, 0xffff8000, v7
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v0, v3
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v3
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v3, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v3, 0xffff8000, v3
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v6, v3
-; GFX8-NEXT: v_sub_u16_e32 v3, v6, v3
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i16_e32 v6, 15, v3
-; GFX8-NEXT: v_xor_b32_e32 v6, 0xffff8000, v6
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v1, v4
-; GFX8-NEXT: v_sub_u16_e32 v1, v1, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v2
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v4, v3
-; GFX8-NEXT: v_sub_u16_e32 v3, v4, v3
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v3
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v2, v5
-; GFX8-NEXT: v_sub_u16_e32 v2, v2, v5
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_e32 v6, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v6, 0x8001, v6
+; GFX8-NEXT: v_min_i16_e32 v7, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_max_i16_e32 v6, v6, v3
+; GFX8-NEXT: v_min_i16_e32 v6, v6, v7
+; GFX8-NEXT: v_mov_b32_e32 v7, -1
+; GFX8-NEXT: v_max_i16_sdwa v8, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v8, 0x8001, v8
+; GFX8-NEXT: v_min_i16_sdwa v9, v0, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_max_i16_sdwa v3, v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v8, -1, v1
+; GFX8-NEXT: v_min_i16_e32 v3, v3, v9
+; GFX8-NEXT: v_add_u16_e32 v8, 0x8001, v8
+; GFX8-NEXT: v_min_i16_e32 v9, -1, v1
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_max_i16_e32 v8, v8, v4
+; GFX8-NEXT: v_min_i16_e32 v8, v8, v9
+; GFX8-NEXT: v_max_i16_sdwa v9, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8001, v9
+; GFX8-NEXT: v_min_i16_sdwa v10, v1, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v10, 0x8000, v10
+; GFX8-NEXT: v_max_i16_sdwa v4, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v9, -1, v2
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v10
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8001, v9
+; GFX8-NEXT: v_min_i16_e32 v10, -1, v2
+; GFX8-NEXT: v_add_u16_e32 v10, 0x8000, v10
+; GFX8-NEXT: v_max_i16_e32 v9, v9, v5
+; GFX8-NEXT: v_min_i16_e32 v9, v9, v10
+; GFX8-NEXT: v_max_i16_sdwa v10, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v10, 0x8001, v10
+; GFX8-NEXT: v_min_i16_sdwa v7, v2, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v7, 0x8000, v7
+; GFX8-NEXT: v_max_i16_sdwa v5, v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_min_i16_e32 v5, v5, v7
+; GFX8-NEXT: v_sub_u16_e32 v6, v0, v6
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v3, v1, v8
+; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_sub_u16_e32 v3, v2, v9
+; GFX8-NEXT: v_sub_u16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v6i16:
@@ -3301,147 +3731,178 @@ define <3 x float> @v_ssubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
define amdgpu_ps <3 x i32> @s_ssubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inreg %rhs) {
; GFX6-LABEL: s_ssubsat_v6i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s10, s5, 16
-; GFX6-NEXT: s_ashr_i32 s11, s2, 16
-; GFX6-NEXT: s_sext_i32_i16 s5, s5
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: s_sub_i32 s10, s11, s10
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: s_sub_i32 s2, s2, s5
-; GFX6-NEXT: s_ashr_i32 s8, s4, 16
-; GFX6-NEXT: s_ashr_i32 s9, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s4, s4
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: v_med3_i32 v2, s10, v0, v1
-; GFX6-NEXT: v_med3_i32 v3, s2, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: s_sub_i32 s2, s9, s8
-; GFX6-NEXT: s_sub_i32 s1, s1, s4
-; GFX6-NEXT: s_ashr_i32 s6, s3, 16
-; GFX6-NEXT: s_ashr_i32 s7, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_med3_i32 v3, s2, v0, v1
-; GFX6-NEXT: v_med3_i32 v4, s1, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX6-NEXT: s_sub_i32 s1, s7, s6
+; GFX6-NEXT: s_lshr_b32 s6, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s12, s0, -1
+; GFX6-NEXT: s_lshr_b32 s9, s3, 16
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_add_i32 s12, s12, 0x80000001
+; GFX6-NEXT: s_min_i32 s13, s0, -1
+; GFX6-NEXT: s_add_i32 s13, s13, 0x80000000
+; GFX6-NEXT: s_max_i32 s3, s12, s3
+; GFX6-NEXT: s_min_i32 s3, s3, s13
; GFX6-NEXT: s_sub_i32 s0, s0, s3
-; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX6-NEXT: v_med3_i32 v4, s1, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s1, v3
-; GFX6-NEXT: v_readfirstlane_b32 s2, v2
+; GFX6-NEXT: s_lshl_b32 s3, s6, 16
+; GFX6-NEXT: s_lshl_b32 s6, s9, 16
+; GFX6-NEXT: s_max_i32 s9, s3, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000001
+; GFX6-NEXT: s_min_i32 s12, s3, -1
+; GFX6-NEXT: s_add_i32 s12, s12, 0x80000000
+; GFX6-NEXT: s_max_i32 s6, s9, s6
+; GFX6-NEXT: s_lshr_b32 s7, s1, 16
+; GFX6-NEXT: s_min_i32 s6, s6, s12
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_sub_i32 s3, s3, s6
+; GFX6-NEXT: s_max_i32 s6, s1, -1
+; GFX6-NEXT: s_lshr_b32 s10, s4, 16
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000001
+; GFX6-NEXT: s_min_i32 s9, s1, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000000
+; GFX6-NEXT: s_max_i32 s4, s6, s4
+; GFX6-NEXT: s_min_i32 s4, s4, s9
+; GFX6-NEXT: s_sub_i32 s1, s1, s4
+; GFX6-NEXT: s_lshl_b32 s4, s7, 16
+; GFX6-NEXT: s_max_i32 s7, s4, -1
+; GFX6-NEXT: s_lshl_b32 s6, s10, 16
+; GFX6-NEXT: s_add_i32 s7, s7, 0x80000001
+; GFX6-NEXT: s_min_i32 s9, s4, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000000
+; GFX6-NEXT: s_max_i32 s6, s7, s6
+; GFX6-NEXT: s_lshr_b32 s8, s2, 16
+; GFX6-NEXT: s_min_i32 s6, s6, s9
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_sub_i32 s4, s4, s6
+; GFX6-NEXT: s_max_i32 s6, s2, -1
+; GFX6-NEXT: s_lshr_b32 s11, s5, 16
+; GFX6-NEXT: s_lshl_b32 s5, s5, 16
+; GFX6-NEXT: s_add_i32 s6, s6, 0x80000001
+; GFX6-NEXT: s_min_i32 s7, s2, -1
+; GFX6-NEXT: s_add_i32 s7, s7, 0x80000000
+; GFX6-NEXT: s_max_i32 s5, s6, s5
+; GFX6-NEXT: s_min_i32 s5, s5, s7
+; GFX6-NEXT: s_sub_i32 s2, s2, s5
+; GFX6-NEXT: s_lshl_b32 s5, s8, 16
+; GFX6-NEXT: s_max_i32 s7, s5, -1
+; GFX6-NEXT: s_ashr_i32 s3, s3, 16
+; GFX6-NEXT: s_lshl_b32 s6, s11, 16
+; GFX6-NEXT: s_add_i32 s7, s7, 0x80000001
+; GFX6-NEXT: s_min_i32 s8, s5, -1
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000000
+; GFX6-NEXT: s_max_i32 s6, s7, s6
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_ashr_i32 s4, s4, 16
+; GFX6-NEXT: s_min_i32 s6, s6, s8
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_sub_i32 s5, s5, s6
+; GFX6-NEXT: s_or_b32 s0, s0, s3
+; GFX6-NEXT: s_and_b32 s3, s4, 0xffff
+; GFX6-NEXT: s_ashr_i32 s5, s5, 16
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_ashr_i32 s2, s2, 16
+; GFX6-NEXT: s_or_b32 s1, s1, s3
+; GFX6-NEXT: s_and_b32 s3, s5, 0xffff
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s3
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ssubsat_v6i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s2, 16
-; GFX8-NEXT: s_sub_i32 s10, s7, s6
-; GFX8-NEXT: s_lshl_b32 s8, s10, 16
-; GFX8-NEXT: s_ashr_i32 s6, s5, 16
-; GFX8-NEXT: s_ashr_i32 s7, s2, 16
-; GFX8-NEXT: s_cmp_lt_i32 s7, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_sext_i32_i16 s8, s10
-; GFX8-NEXT: s_lshr_b32 s8, s8, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s6, s8, s10
-; GFX8-NEXT: s_sub_i32 s11, s2, s5
-; GFX8-NEXT: s_lshl_b32 s10, s6, 16
-; GFX8-NEXT: s_lshl_b32 s8, s11, 16
-; GFX8-NEXT: s_sext_i32_i16 s5, s5
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_cmp_lt_i32 s2, s5
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_sext_i32_i16 s2, s11
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 15
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s2, s2, s11
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_lshr_b32 s6, s1, 16
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_sub_i32 s5, s6, s5
-; GFX8-NEXT: s_or_b32 s2, s2, s10
-; GFX8-NEXT: s_lshl_b32 s8, s5, 16
-; GFX8-NEXT: s_ashr_i32 s6, s4, 16
-; GFX8-NEXT: s_ashr_i32 s7, s1, 16
-; GFX8-NEXT: s_cmp_lt_i32 s7, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_sext_i32_i16 s8, s5
-; GFX8-NEXT: s_lshr_b32 s8, s8, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s5, s8, s5
-; GFX8-NEXT: s_sub_i32 s9, s1, s4
-; GFX8-NEXT: s_lshl_b32 s8, s5, 16
-; GFX8-NEXT: s_lshl_b32 s6, s9, 16
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_cmp_lt_i32 s1, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_sext_i32_i16 s1, s9
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s9
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
-; GFX8-NEXT: s_lshr_b32 s5, s0, 16
-; GFX8-NEXT: s_or_b32 s1, s1, s8
-; GFX8-NEXT: s_sub_i32 s8, s5, s4
-; GFX8-NEXT: s_lshl_b32 s6, s8, 16
-; GFX8-NEXT: s_ashr_i32 s4, s3, 16
-; GFX8-NEXT: s_ashr_i32 s5, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s5, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_sext_i32_i16 s6, s8
-; GFX8-NEXT: s_lshr_b32 s6, s6, 15
-; GFX8-NEXT: s_xor_b32 s6, s6, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s4, s6, s8
-; GFX8-NEXT: s_sub_i32 s9, s0, s3
-; GFX8-NEXT: s_lshl_b32 s8, s4, 16
-; GFX8-NEXT: s_lshl_b32 s6, s9, 16
+; GFX8-NEXT: s_sext_i32_i16 s12, s0
+; GFX8-NEXT: s_max_i32 s13, s12, -1
+; GFX8-NEXT: s_add_i32 s13, s13, 0x8001
+; GFX8-NEXT: s_lshr_b32 s9, s3, 16
+; GFX8-NEXT: s_min_i32 s12, s12, -1
+; GFX8-NEXT: s_sext_i32_i16 s13, s13
; GFX8-NEXT: s_sext_i32_i16 s3, s3
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s0, s3
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_sext_i32_i16 s0, s9
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s0, s0, s9
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s8
+; GFX8-NEXT: s_add_i32 s12, s12, 0x8000
+; GFX8-NEXT: s_max_i32 s3, s13, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: s_min_i32 s3, s3, s12
+; GFX8-NEXT: s_sub_i32 s0, s0, s3
+; GFX8-NEXT: s_sext_i32_i16 s3, s6
+; GFX8-NEXT: s_max_i32 s12, s3, -1
+; GFX8-NEXT: s_add_i32 s12, s12, 0x8001
+; GFX8-NEXT: s_min_i32 s3, s3, -1
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_add_i32 s3, s3, 0x8000
+; GFX8-NEXT: s_max_i32 s9, s12, s9
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sext_i32_i16 s3, s3
+; GFX8-NEXT: s_min_i32 s3, s9, s3
+; GFX8-NEXT: s_sub_i32 s3, s6, s3
+; GFX8-NEXT: s_sext_i32_i16 s6, s1
+; GFX8-NEXT: s_max_i32 s9, s6, -1
+; GFX8-NEXT: s_add_i32 s9, s9, 0x8001
+; GFX8-NEXT: s_lshr_b32 s10, s4, 16
+; GFX8-NEXT: s_min_i32 s6, s6, -1
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8000
+; GFX8-NEXT: s_max_i32 s4, s9, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: s_min_i32 s4, s4, s6
+; GFX8-NEXT: s_sub_i32 s1, s1, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s7
+; GFX8-NEXT: s_max_i32 s6, s4, -1
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8001
+; GFX8-NEXT: s_min_i32 s4, s4, -1
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s9, s10
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8000
+; GFX8-NEXT: s_max_i32 s6, s6, s9
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_min_i32 s4, s6, s4
+; GFX8-NEXT: s_sext_i32_i16 s6, s2
+; GFX8-NEXT: s_sub_i32 s4, s7, s4
+; GFX8-NEXT: s_max_i32 s7, s6, -1
+; GFX8-NEXT: s_add_i32 s7, s7, 0x8001
+; GFX8-NEXT: s_lshr_b32 s11, s5, 16
+; GFX8-NEXT: s_min_i32 s6, s6, -1
+; GFX8-NEXT: s_sext_i32_i16 s7, s7
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8000
+; GFX8-NEXT: s_max_i32 s5, s7, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_lshr_b32 s8, s2, 16
+; GFX8-NEXT: s_min_i32 s5, s5, s6
+; GFX8-NEXT: s_sub_i32 s2, s2, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s8
+; GFX8-NEXT: s_max_i32 s6, s5, -1
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8001
+; GFX8-NEXT: s_min_i32 s5, s5, -1
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s7, s11
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8000
+; GFX8-NEXT: s_max_i32 s6, s6, s7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_min_i32 s5, s6, s5
+; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s4
+; GFX8-NEXT: s_sub_i32 s5, s8, s5
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s5
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_v6i16:
@@ -3475,129 +3936,178 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
; GFX6-LABEL: v_ssubsat_v8i16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_ashrrev_i32_e32 v12, 16, v5
-; GFX6-NEXT: v_ashrrev_i32_e32 v13, 16, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v14, 16, v4
-; GFX6-NEXT: v_ashrrev_i32_e32 v15, 16, v0
-; GFX6-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX6-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX6-NEXT: v_sub_i32_e32 v14, vcc, v15, v14
-; GFX6-NEXT: s_movk_i32 s4, 0x8000
-; GFX6-NEXT: v_mov_b32_e32 v15, 0x7fff
+; GFX6-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v16, -1, v0
+; GFX6-NEXT: v_mov_b32_e32 v17, 0x80000001
+; GFX6-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_add_i32_e32 v16, vcc, v16, v17
+; GFX6-NEXT: v_min_i32_e32 v18, -1, v0
+; GFX6-NEXT: v_bfrev_b32_e32 v19, 1
+; GFX6-NEXT: v_add_i32_e32 v18, vcc, v18, v19
+; GFX6-NEXT: v_max_i32_e32 v4, v16, v4
+; GFX6-NEXT: v_min_i32_e32 v4, v4, v18
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v13, v12
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v8
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; GFX6-NEXT: v_max_i32_e32 v12, -1, v4
+; GFX6-NEXT: v_add_i32_e32 v12, vcc, v12, v17
+; GFX6-NEXT: v_min_i32_e32 v16, -1, v4
+; GFX6-NEXT: v_add_i32_e32 v16, vcc, v16, v19
+; GFX6-NEXT: v_max_i32_e32 v8, v12, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v16
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v4, v8
+; GFX6-NEXT: v_max_i32_e32 v8, -1, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v17
+; GFX6-NEXT: v_min_i32_e32 v12, -1, v1
+; GFX6-NEXT: v_add_i32_e32 v12, vcc, v12, v19
+; GFX6-NEXT: v_max_i32_e32 v5, v8, v5
+; GFX6-NEXT: v_min_i32_e32 v5, v5, v12
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v5
-; GFX6-NEXT: v_med3_i32 v4, v4, s4, v15
-; GFX6-NEXT: v_med3_i32 v1, v1, s4, v15
-; GFX6-NEXT: v_ashrrev_i32_e32 v10, 16, v6
-; GFX6-NEXT: v_ashrrev_i32_e32 v11, 16, v2
-; GFX6-NEXT: v_bfe_i32 v6, v6, 0, 16
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v9
+; GFX6-NEXT: v_max_i32_e32 v9, -1, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v13
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT: v_min_i32_e32 v12, -1, v5
+; GFX6-NEXT: v_add_i32_e32 v12, vcc, v12, v19
+; GFX6-NEXT: v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v12
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, v5, v8
+; GFX6-NEXT: v_max_i32_e32 v8, -1, v2
+; GFX6-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v17
+; GFX6-NEXT: v_min_i32_e32 v9, -1, v2
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v19
+; GFX6-NEXT: v_max_i32_e32 v6, v8, v6
+; GFX6-NEXT: v_min_i32_e32 v6, v6, v9
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v6, 16, v10
+; GFX6-NEXT: v_max_i32_e32 v9, -1, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v14
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT: v_min_i32_e32 v10, -1, v6
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT: v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v10
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_sub_i32_e32 v6, vcc, v6, v8
+; GFX6-NEXT: v_max_i32_e32 v8, -1, v3
+; GFX6-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX6-NEXT: v_add_i32_e32 v8, vcc, v8, v17
+; GFX6-NEXT: v_min_i32_e32 v9, -1, v3
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v19
+; GFX6-NEXT: v_max_i32_e32 v7, v8, v7
+; GFX6-NEXT: v_min_i32_e32 v7, v7, v9
+; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v4
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v7
+; GFX6-NEXT: v_lshlrev_b32_e32 v7, 16, v11
+; GFX6-NEXT: v_ashrrev_i32_e32 v0, 16, v0
+; GFX6-NEXT: v_max_i32_e32 v9, -1, v7
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v8, 16, v15
+; GFX6-NEXT: v_add_i32_e32 v9, vcc, v9, v17
+; GFX6-NEXT: v_min_i32_e32 v10, -1, v7
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v1, 16, v1
+; GFX6-NEXT: v_add_i32_e32 v10, vcc, v10, v19
+; GFX6-NEXT: v_max_i32_e32 v8, v9, v8
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX6-NEXT: v_ashrrev_i32_e32 v6, 16, v6
+; GFX6-NEXT: v_min_i32_e32 v8, v8, v10
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v11, v10
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
-; GFX6-NEXT: v_med3_i32 v4, v4, s4, v15
-; GFX6-NEXT: v_med3_i32 v2, v2, s4, v15
-; GFX6-NEXT: v_ashrrev_i32_e32 v8, 16, v7
-; GFX6-NEXT: v_ashrrev_i32_e32 v9, 16, v3
-; GFX6-NEXT: v_bfe_i32 v7, v7, 0, 16
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v2, 16, v2
+; GFX6-NEXT: v_sub_i32_e32 v7, vcc, v7, v8
+; GFX6-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v7
; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v9, v8
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v7
-; GFX6-NEXT: v_med3_i32 v14, v14, s4, v15
-; GFX6-NEXT: v_med3_i32 v0, v0, s4, v15
-; GFX6-NEXT: v_med3_i32 v4, v4, s4, v15
-; GFX6-NEXT: v_med3_i32 v3, v3, s4, v15
-; GFX6-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v7
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v14
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_ssubsat_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v9, v8
-; GFX8-NEXT: v_sub_u16_e32 v8, v9, v8
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v8
-; GFX8-NEXT: v_ashrrev_i16_e32 v9, 15, v8
-; GFX8-NEXT: v_xor_b32_e32 v9, 0xffff8000, v9
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v8, v8, v9, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v0, v4
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v0
-; GFX8-NEXT: v_ashrrev_i16_e32 v4, 15, v0
-; GFX8-NEXT: v_xor_b32_e32 v4, 0xffff8000, v4
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v1
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v8, v4
-; GFX8-NEXT: v_sub_u16_e32 v4, v8, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v8, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v8, 0xffff8000, v8
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v1, v5
-; GFX8-NEXT: v_sub_u16_e32 v1, v1, v5
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v1
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 15, v1
-; GFX8-NEXT: v_xor_b32_e32 v5, 0xffff8000, v5
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc
-; GFX8-NEXT: v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v5, v4
-; GFX8-NEXT: v_sub_u16_e32 v4, v5, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v5, 0xffff8000, v5
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v2, v6
-; GFX8-NEXT: v_sub_u16_e32 v2, v2, v6
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v2
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 15, v2
-; GFX8-NEXT: v_xor_b32_e32 v5, 0xffff8000, v5
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc
-; GFX8-NEXT: v_or_b32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v7
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v5, v4
-; GFX8-NEXT: v_sub_u16_e32 v4, v5, v4
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v4
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 15, v4
-; GFX8-NEXT: v_xor_b32_e32 v5, 0xffff8000, v5
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc
-; GFX8-NEXT: v_cmp_lt_i16_e32 vcc, v3, v7
-; GFX8-NEXT: v_sub_u16_e32 v3, v3, v7
-; GFX8-NEXT: v_cmp_gt_i16_e64 s[4:5], 0, v3
-; GFX8-NEXT: v_ashrrev_i16_e32 v5, 15, v3
-; GFX8-NEXT: v_xor_b32_e32 v5, 0xffff8000, v5
-; GFX8-NEXT: s_xor_b64 vcc, vcc, s[4:5]
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
-; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_max_i16_e32 v8, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v8, 0x8001, v8
+; GFX8-NEXT: v_min_i16_e32 v9, -1, v0
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_max_i16_e32 v8, v8, v4
+; GFX8-NEXT: v_min_i16_e32 v8, v8, v9
+; GFX8-NEXT: v_mov_b32_e32 v9, -1
+; GFX8-NEXT: v_max_i16_sdwa v10, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v10, 0x8001, v10
+; GFX8-NEXT: v_min_i16_sdwa v11, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v11, 0x8000, v11
+; GFX8-NEXT: v_max_i16_sdwa v4, v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v10, -1, v1
+; GFX8-NEXT: v_min_i16_e32 v4, v4, v11
+; GFX8-NEXT: v_add_u16_e32 v10, 0x8001, v10
+; GFX8-NEXT: v_min_i16_e32 v11, -1, v1
+; GFX8-NEXT: v_add_u16_e32 v11, 0x8000, v11
+; GFX8-NEXT: v_max_i16_e32 v10, v10, v5
+; GFX8-NEXT: v_min_i16_e32 v10, v10, v11
+; GFX8-NEXT: v_max_i16_sdwa v11, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v11, 0x8001, v11
+; GFX8-NEXT: v_min_i16_sdwa v12, v1, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v12, 0x8000, v12
+; GFX8-NEXT: v_max_i16_sdwa v5, v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v11, -1, v2
+; GFX8-NEXT: v_min_i16_e32 v5, v5, v12
+; GFX8-NEXT: v_add_u16_e32 v11, 0x8001, v11
+; GFX8-NEXT: v_min_i16_e32 v12, -1, v2
+; GFX8-NEXT: v_add_u16_e32 v12, 0x8000, v12
+; GFX8-NEXT: v_max_i16_e32 v11, v11, v6
+; GFX8-NEXT: v_min_i16_e32 v11, v11, v12
+; GFX8-NEXT: v_max_i16_sdwa v12, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v12, 0x8001, v12
+; GFX8-NEXT: v_min_i16_sdwa v13, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v13, 0x8000, v13
+; GFX8-NEXT: v_max_i16_sdwa v6, v12, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_max_i16_e32 v12, -1, v3
+; GFX8-NEXT: v_min_i16_e32 v6, v6, v13
+; GFX8-NEXT: v_add_u16_e32 v12, 0x8001, v12
+; GFX8-NEXT: v_min_i16_e32 v13, -1, v3
+; GFX8-NEXT: v_add_u16_e32 v13, 0x8000, v13
+; GFX8-NEXT: v_max_i16_e32 v12, v12, v7
+; GFX8-NEXT: v_min_i16_e32 v12, v12, v13
+; GFX8-NEXT: v_max_i16_sdwa v13, v3, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v13, 0x8001, v13
+; GFX8-NEXT: v_min_i16_sdwa v9, v3, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_add_u16_e32 v9, 0x8000, v9
+; GFX8-NEXT: v_max_i16_sdwa v7, v13, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e32 v8, v0, v8
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_sub_u16_e32 v4, v1, v10
+; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_min_i16_e32 v7, v7, v9
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_sub_u16_e32 v4, v2, v11
+; GFX8-NEXT: v_sub_u16_sdwa v2, v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_sub_u16_e32 v4, v3, v12
+; GFX8-NEXT: v_sub_u16_sdwa v3, v3, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_ssubsat_v8i16:
@@ -3625,192 +4135,234 @@ define <4 x float> @v_ssubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
define amdgpu_ps <4 x i32> @s_ssubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inreg %rhs) {
; GFX6-LABEL: s_ssubsat_v8i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_ashr_i32 s14, s7, 16
-; GFX6-NEXT: s_ashr_i32 s15, s3, 16
-; GFX6-NEXT: s_sext_i32_i16 s7, s7
-; GFX6-NEXT: s_sext_i32_i16 s3, s3
-; GFX6-NEXT: s_sub_i32 s14, s15, s14
-; GFX6-NEXT: v_mov_b32_e32 v0, 0xffff8000
-; GFX6-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX6-NEXT: s_sub_i32 s3, s3, s7
-; GFX6-NEXT: s_ashr_i32 s12, s6, 16
-; GFX6-NEXT: s_ashr_i32 s13, s2, 16
-; GFX6-NEXT: s_sext_i32_i16 s6, s6
-; GFX6-NEXT: s_sext_i32_i16 s2, s2
-; GFX6-NEXT: v_med3_i32 v2, s14, v0, v1
-; GFX6-NEXT: v_med3_i32 v3, s3, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX6-NEXT: s_sub_i32 s3, s13, s12
-; GFX6-NEXT: s_sub_i32 s2, s2, s6
-; GFX6-NEXT: s_ashr_i32 s10, s5, 16
-; GFX6-NEXT: s_ashr_i32 s11, s1, 16
-; GFX6-NEXT: s_sext_i32_i16 s5, s5
-; GFX6-NEXT: s_sext_i32_i16 s1, s1
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_med3_i32 v3, s3, v0, v1
-; GFX6-NEXT: v_med3_i32 v4, s2, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX6-NEXT: s_sub_i32 s2, s11, s10
-; GFX6-NEXT: s_sub_i32 s1, s1, s5
-; GFX6-NEXT: s_ashr_i32 s8, s4, 16
-; GFX6-NEXT: s_ashr_i32 s9, s0, 16
-; GFX6-NEXT: s_sext_i32_i16 s4, s4
-; GFX6-NEXT: s_sext_i32_i16 s0, s0
-; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX6-NEXT: v_med3_i32 v4, s2, v0, v1
-; GFX6-NEXT: v_med3_i32 v5, s1, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX6-NEXT: s_sub_i32 s1, s9, s8
+; GFX6-NEXT: s_lshr_b32 s8, s0, 16
+; GFX6-NEXT: s_lshl_b32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s16, s0, -1
+; GFX6-NEXT: s_lshr_b32 s12, s4, 16
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_add_i32 s16, s16, 0x80000001
+; GFX6-NEXT: s_min_i32 s17, s0, -1
+; GFX6-NEXT: s_add_i32 s17, s17, 0x80000000
+; GFX6-NEXT: s_max_i32 s4, s16, s4
+; GFX6-NEXT: s_min_i32 s4, s4, s17
; GFX6-NEXT: s_sub_i32 s0, s0, s4
-; GFX6-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX6-NEXT: v_med3_i32 v5, s1, v0, v1
-; GFX6-NEXT: v_med3_i32 v0, s0, v0, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX6-NEXT: v_readfirstlane_b32 s0, v0
-; GFX6-NEXT: v_readfirstlane_b32 s1, v4
-; GFX6-NEXT: v_readfirstlane_b32 s2, v3
-; GFX6-NEXT: v_readfirstlane_b32 s3, v2
+; GFX6-NEXT: s_lshl_b32 s4, s8, 16
+; GFX6-NEXT: s_lshl_b32 s8, s12, 16
+; GFX6-NEXT: s_max_i32 s12, s4, -1
+; GFX6-NEXT: s_add_i32 s12, s12, 0x80000001
+; GFX6-NEXT: s_min_i32 s16, s4, -1
+; GFX6-NEXT: s_add_i32 s16, s16, 0x80000000
+; GFX6-NEXT: s_max_i32 s8, s12, s8
+; GFX6-NEXT: s_lshr_b32 s9, s1, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s16
+; GFX6-NEXT: s_lshl_b32 s1, s1, 16
+; GFX6-NEXT: s_sub_i32 s4, s4, s8
+; GFX6-NEXT: s_max_i32 s8, s1, -1
+; GFX6-NEXT: s_lshr_b32 s13, s5, 16
+; GFX6-NEXT: s_lshl_b32 s5, s5, 16
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000001
+; GFX6-NEXT: s_min_i32 s12, s1, -1
+; GFX6-NEXT: s_add_i32 s12, s12, 0x80000000
+; GFX6-NEXT: s_max_i32 s5, s8, s5
+; GFX6-NEXT: s_min_i32 s5, s5, s12
+; GFX6-NEXT: s_sub_i32 s1, s1, s5
+; GFX6-NEXT: s_lshl_b32 s5, s9, 16
+; GFX6-NEXT: s_max_i32 s9, s5, -1
+; GFX6-NEXT: s_lshl_b32 s8, s13, 16
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000001
+; GFX6-NEXT: s_min_i32 s12, s5, -1
+; GFX6-NEXT: s_add_i32 s12, s12, 0x80000000
+; GFX6-NEXT: s_max_i32 s8, s9, s8
+; GFX6-NEXT: s_lshr_b32 s10, s2, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s12
+; GFX6-NEXT: s_lshl_b32 s2, s2, 16
+; GFX6-NEXT: s_sub_i32 s5, s5, s8
+; GFX6-NEXT: s_max_i32 s8, s2, -1
+; GFX6-NEXT: s_lshr_b32 s14, s6, 16
+; GFX6-NEXT: s_lshl_b32 s6, s6, 16
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000001
+; GFX6-NEXT: s_min_i32 s9, s2, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000000
+; GFX6-NEXT: s_max_i32 s6, s8, s6
+; GFX6-NEXT: s_min_i32 s6, s6, s9
+; GFX6-NEXT: s_sub_i32 s2, s2, s6
+; GFX6-NEXT: s_lshl_b32 s6, s10, 16
+; GFX6-NEXT: s_max_i32 s9, s6, -1
+; GFX6-NEXT: s_lshl_b32 s8, s14, 16
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000001
+; GFX6-NEXT: s_min_i32 s10, s6, -1
+; GFX6-NEXT: s_add_i32 s10, s10, 0x80000000
+; GFX6-NEXT: s_max_i32 s8, s9, s8
+; GFX6-NEXT: s_lshr_b32 s11, s3, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s10
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_sub_i32 s6, s6, s8
+; GFX6-NEXT: s_max_i32 s8, s3, -1
+; GFX6-NEXT: s_lshr_b32 s15, s7, 16
+; GFX6-NEXT: s_lshl_b32 s7, s7, 16
+; GFX6-NEXT: s_add_i32 s8, s8, 0x80000001
+; GFX6-NEXT: s_min_i32 s9, s3, -1
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000000
+; GFX6-NEXT: s_max_i32 s7, s8, s7
+; GFX6-NEXT: s_min_i32 s7, s7, s9
+; GFX6-NEXT: s_ashr_i32 s4, s4, 16
+; GFX6-NEXT: s_sub_i32 s3, s3, s7
+; GFX6-NEXT: s_lshl_b32 s7, s11, 16
+; GFX6-NEXT: s_ashr_i32 s0, s0, 16
+; GFX6-NEXT: s_max_i32 s9, s7, -1
+; GFX6-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX6-NEXT: s_ashr_i32 s5, s5, 16
+; GFX6-NEXT: s_lshl_b32 s8, s15, 16
+; GFX6-NEXT: s_add_i32 s9, s9, 0x80000001
+; GFX6-NEXT: s_min_i32 s10, s7, -1
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_ashr_i32 s1, s1, 16
+; GFX6-NEXT: s_add_i32 s10, s10, 0x80000000
+; GFX6-NEXT: s_max_i32 s8, s9, s8
+; GFX6-NEXT: s_or_b32 s0, s0, s4
+; GFX6-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX6-NEXT: s_ashr_i32 s6, s6, 16
+; GFX6-NEXT: s_min_i32 s8, s8, s10
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_ashr_i32 s2, s2, 16
+; GFX6-NEXT: s_sub_i32 s7, s7, s8
+; GFX6-NEXT: s_or_b32 s1, s1, s4
+; GFX6-NEXT: s_and_b32 s4, s6, 0xffff
+; GFX6-NEXT: s_ashr_i32 s7, s7, 16
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_ashr_i32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s2, s2, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_lshl_b32 s4, s4, 16
+; GFX6-NEXT: s_or_b32 s3, s3, s4
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_ssubsat_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s8, s7, 16
-; GFX8-NEXT: s_lshr_b32 s9, s3, 16
-; GFX8-NEXT: s_sub_i32 s12, s9, s8
-; GFX8-NEXT: s_lshl_b32 s10, s12, 16
-; GFX8-NEXT: s_ashr_i32 s8, s7, 16
-; GFX8-NEXT: s_ashr_i32 s9, s3, 16
-; GFX8-NEXT: s_cmp_lt_i32 s9, s8
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, 0
-; GFX8-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX8-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; GFX8-NEXT: s_sext_i32_i16 s10, s12
-; GFX8-NEXT: s_lshr_b32 s10, s10, 15
-; GFX8-NEXT: s_xor_b32 s10, s10, 0x8000
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT: s_cselect_b32 s8, s10, s12
-; GFX8-NEXT: s_sub_i32 s13, s3, s7
-; GFX8-NEXT: s_lshl_b32 s12, s8, 16
-; GFX8-NEXT: s_lshl_b32 s10, s13, 16
-; GFX8-NEXT: s_sext_i32_i16 s7, s7
-; GFX8-NEXT: s_sext_i32_i16 s3, s3
-; GFX8-NEXT: s_cmp_lt_i32 s3, s7
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, 0
-; GFX8-NEXT: s_sext_i32_i16 s3, s13
-; GFX8-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX8-NEXT: s_lshr_b32 s3, s3, 15
-; GFX8-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; GFX8-NEXT: s_xor_b32 s3, s3, 0x8000
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT: s_cselect_b32 s3, s3, s13
-; GFX8-NEXT: s_lshr_b32 s7, s6, 16
-; GFX8-NEXT: s_lshr_b32 s8, s2, 16
-; GFX8-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX8-NEXT: s_sub_i32 s7, s8, s7
-; GFX8-NEXT: s_or_b32 s3, s3, s12
-; GFX8-NEXT: s_lshl_b32 s10, s7, 16
-; GFX8-NEXT: s_ashr_i32 s8, s6, 16
-; GFX8-NEXT: s_ashr_i32 s9, s2, 16
-; GFX8-NEXT: s_cmp_lt_i32 s9, s8
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s10, 0
-; GFX8-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GFX8-NEXT: s_xor_b64 s[8:9], s[8:9], s[10:11]
-; GFX8-NEXT: s_sext_i32_i16 s10, s7
-; GFX8-NEXT: s_lshr_b32 s10, s10, 15
-; GFX8-NEXT: s_xor_b32 s10, s10, 0x8000
-; GFX8-NEXT: s_and_b64 s[8:9], s[8:9], exec
-; GFX8-NEXT: s_cselect_b32 s7, s10, s7
-; GFX8-NEXT: s_sub_i32 s11, s2, s6
-; GFX8-NEXT: s_lshl_b32 s10, s7, 16
-; GFX8-NEXT: s_lshl_b32 s8, s11, 16
-; GFX8-NEXT: s_sext_i32_i16 s6, s6
-; GFX8-NEXT: s_sext_i32_i16 s2, s2
-; GFX8-NEXT: s_cmp_lt_i32 s2, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_sext_i32_i16 s2, s11
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_lshr_b32 s2, s2, 15
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s2, s2, s11
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s1, 16
-; GFX8-NEXT: s_or_b32 s2, s2, s10
-; GFX8-NEXT: s_sub_i32 s10, s7, s6
-; GFX8-NEXT: s_lshl_b32 s8, s10, 16
-; GFX8-NEXT: s_ashr_i32 s6, s5, 16
-; GFX8-NEXT: s_ashr_i32 s7, s1, 16
-; GFX8-NEXT: s_cmp_lt_i32 s7, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_sext_i32_i16 s8, s10
-; GFX8-NEXT: s_lshr_b32 s8, s8, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s6, s8, s10
-; GFX8-NEXT: s_sub_i32 s11, s1, s5
-; GFX8-NEXT: s_lshl_b32 s10, s6, 16
-; GFX8-NEXT: s_lshl_b32 s8, s11, 16
-; GFX8-NEXT: s_sext_i32_i16 s5, s5
-; GFX8-NEXT: s_sext_i32_i16 s1, s1
-; GFX8-NEXT: s_cmp_lt_i32 s1, s5
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_sext_i32_i16 s1, s11
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_lshr_b32 s1, s1, 15
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s1, s1, s11
-; GFX8-NEXT: s_lshr_b32 s5, s4, 16
-; GFX8-NEXT: s_lshr_b32 s6, s0, 16
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX8-NEXT: s_sub_i32 s5, s6, s5
-; GFX8-NEXT: s_or_b32 s1, s1, s10
-; GFX8-NEXT: s_lshl_b32 s8, s5, 16
-; GFX8-NEXT: s_ashr_i32 s6, s4, 16
-; GFX8-NEXT: s_ashr_i32 s7, s0, 16
-; GFX8-NEXT: s_cmp_lt_i32 s7, s6
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s8, 0
-; GFX8-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX8-NEXT: s_xor_b64 s[6:7], s[6:7], s[8:9]
-; GFX8-NEXT: s_sext_i32_i16 s8, s5
-; GFX8-NEXT: s_lshr_b32 s8, s8, 15
-; GFX8-NEXT: s_xor_b32 s8, s8, 0x8000
-; GFX8-NEXT: s_and_b64 s[6:7], s[6:7], exec
-; GFX8-NEXT: s_cselect_b32 s5, s8, s5
-; GFX8-NEXT: s_sub_i32 s9, s0, s4
-; GFX8-NEXT: s_lshl_b32 s8, s5, 16
-; GFX8-NEXT: s_lshl_b32 s6, s9, 16
+; GFX8-NEXT: s_sext_i32_i16 s16, s0
+; GFX8-NEXT: s_max_i32 s17, s16, -1
+; GFX8-NEXT: s_add_i32 s17, s17, 0x8001
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
+; GFX8-NEXT: s_min_i32 s16, s16, -1
+; GFX8-NEXT: s_sext_i32_i16 s17, s17
; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_sext_i32_i16 s0, s0
-; GFX8-NEXT: s_cmp_lt_i32 s0, s4
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: s_cmp_lt_i32 s6, 0
-; GFX8-NEXT: s_sext_i32_i16 s0, s9
-; GFX8-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX8-NEXT: s_lshr_b32 s0, s0, 15
-; GFX8-NEXT: s_xor_b64 s[4:5], s[4:5], s[6:7]
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x8000
-; GFX8-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX8-NEXT: s_cselect_b32 s0, s0, s9
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s8
+; GFX8-NEXT: s_add_i32 s16, s16, 0x8000
+; GFX8-NEXT: s_max_i32 s4, s17, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_sext_i32_i16 s16, s16
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
+; GFX8-NEXT: s_min_i32 s4, s4, s16
+; GFX8-NEXT: s_sub_i32 s0, s0, s4
+; GFX8-NEXT: s_sext_i32_i16 s4, s8
+; GFX8-NEXT: s_max_i32 s16, s4, -1
+; GFX8-NEXT: s_add_i32 s16, s16, 0x8001
+; GFX8-NEXT: s_min_i32 s4, s4, -1
+; GFX8-NEXT: s_sext_i32_i16 s16, s16
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_add_i32 s4, s4, 0x8000
+; GFX8-NEXT: s_max_i32 s12, s16, s12
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_sext_i32_i16 s4, s4
+; GFX8-NEXT: s_min_i32 s4, s12, s4
+; GFX8-NEXT: s_sub_i32 s4, s8, s4
+; GFX8-NEXT: s_sext_i32_i16 s8, s1
+; GFX8-NEXT: s_max_i32 s12, s8, -1
+; GFX8-NEXT: s_add_i32 s12, s12, 0x8001
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
+; GFX8-NEXT: s_min_i32 s8, s8, -1
+; GFX8-NEXT: s_sext_i32_i16 s12, s12
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8000
+; GFX8-NEXT: s_max_i32 s5, s12, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: s_min_i32 s5, s5, s8
+; GFX8-NEXT: s_sub_i32 s1, s1, s5
+; GFX8-NEXT: s_sext_i32_i16 s5, s9
+; GFX8-NEXT: s_max_i32 s8, s5, -1
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8001
+; GFX8-NEXT: s_min_i32 s5, s5, -1
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s12, s13
+; GFX8-NEXT: s_add_i32 s5, s5, 0x8000
+; GFX8-NEXT: s_max_i32 s8, s8, s12
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s5, s5
+; GFX8-NEXT: s_min_i32 s5, s8, s5
+; GFX8-NEXT: s_sext_i32_i16 s8, s2
+; GFX8-NEXT: s_sub_i32 s5, s9, s5
+; GFX8-NEXT: s_max_i32 s9, s8, -1
+; GFX8-NEXT: s_add_i32 s9, s9, 0x8001
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: s_min_i32 s8, s8, -1
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8000
+; GFX8-NEXT: s_max_i32 s6, s9, s6
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: s_min_i32 s6, s6, s8
+; GFX8-NEXT: s_sub_i32 s2, s2, s6
+; GFX8-NEXT: s_sext_i32_i16 s6, s10
+; GFX8-NEXT: s_max_i32 s8, s6, -1
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8001
+; GFX8-NEXT: s_min_i32 s6, s6, -1
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s9, s14
+; GFX8-NEXT: s_add_i32 s6, s6, 0x8000
+; GFX8-NEXT: s_max_i32 s8, s8, s9
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s6, s6
+; GFX8-NEXT: s_min_i32 s6, s8, s6
+; GFX8-NEXT: s_sext_i32_i16 s8, s3
+; GFX8-NEXT: s_max_i32 s9, s8, -1
+; GFX8-NEXT: s_add_i32 s9, s9, 0x8001
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: s_min_i32 s8, s8, -1
+; GFX8-NEXT: s_sext_i32_i16 s9, s9
+; GFX8-NEXT: s_sext_i32_i16 s7, s7
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8000
+; GFX8-NEXT: s_max_i32 s7, s9, s7
+; GFX8-NEXT: s_sext_i32_i16 s7, s7
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
+; GFX8-NEXT: s_min_i32 s7, s7, s8
+; GFX8-NEXT: s_sub_i32 s3, s3, s7
+; GFX8-NEXT: s_sext_i32_i16 s7, s11
+; GFX8-NEXT: s_max_i32 s8, s7, -1
+; GFX8-NEXT: s_add_i32 s8, s8, 0x8001
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: s_min_i32 s7, s7, -1
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s9, s15
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_add_i32 s7, s7, 0x8000
+; GFX8-NEXT: s_max_i32 s8, s8, s9
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s5
+; GFX8-NEXT: s_sub_i32 s6, s10, s6
+; GFX8-NEXT: s_sext_i32_i16 s8, s8
+; GFX8-NEXT: s_sext_i32_i16 s7, s7
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_min_i32 s7, s8, s7
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX8-NEXT: s_sub_i32 s7, s11, s7
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_ssubsat_v8i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
index ea93a862dcacb..4352b32b30dd4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/strict_fma.f16.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX8 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX9,GFX900 %s
; xUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.42-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX9,GFX942 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX11 %s
@@ -16,6 +16,26 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_fma_f16 v2, s16, v2, v3
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_fma_f16 v2, s0, v2, v3
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -38,24 +58,6 @@ define void @v_constained_fma_f16_fpexcept_strict_uni(half inreg %x, half inreg
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, s16, v2, v3
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, s0, v2, v3
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
ret void
@@ -70,6 +72,24 @@ define void @v_constained_fma_f16_fpexcept_strict_div(half %x, half %y, half %z,
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -87,22 +107,6 @@ define void @v_constained_fma_f16_fpexcept_strict_div(half %x, half %y, half %z,
; GFX12-NEXT: v_fma_f16 v0.l, v0.l, v1.l, v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
ret void
@@ -112,21 +116,46 @@ define void @v_constained_fma_v2f16_fpexcept_strict_uni(<2 x half> inreg %x, <2
; GFX8-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s18, 16
+; GFX8-NEXT: v_mov_b32_e32 v2, s17
+; GFX8-NEXT: v_mov_b32_e32 v3, s18
; GFX8-NEXT: s_lshr_b32 s5, s17, 16
-; GFX8-NEXT: s_lshr_b32 s6, s16, 16
+; GFX8-NEXT: s_lshr_b32 s6, s18, 16
+; GFX8-NEXT: v_fma_f16 v2, s16, v2, v3
+; GFX8-NEXT: s_lshr_b32 s4, s16, 16
+; GFX8-NEXT: v_readfirstlane_b32 s7, v2
; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_fma_f16 v2, s6, v2, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s17
-; GFX8-NEXT: v_mov_b32_e32 v4, s18
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_fma_f16 v3, s16, v3, v4
-; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, s6
+; GFX8-NEXT: v_fma_f16 v2, s4, v2, v3
+; GFX8-NEXT: v_readfirstlane_b32 s4, v2
+; GFX8-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT: s_bfe_u32 s5, s7, 0x100000
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s4, s5, s4
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
+; GFX900-NEXT: global_store_dword v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
+; GFX942-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -156,24 +185,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_uni(<2 x half> inreg %x, <2
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
-; GFX900-NEXT: global_store_dword v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %x, <2 x half> %y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <2 x half> %val, ptr addrspace(1) %out
ret void
@@ -183,17 +194,36 @@ define void @v_constained_fma_v2f16_fpexcept_strict_div(<2 x half> %x, <2 x half
; GFX8-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX8-NEXT: v_fma_f16 v5, v7, v6, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v2
; GFX8-NEXT: v_fma_f16 v0, v0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-NEXT: v_fma_f16 v1, v5, v6, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: flat_store_dword v[3:4], v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2
+; GFX900-NEXT: global_store_dword v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2
+; GFX942-NEXT: global_store_dword v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -211,22 +241,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_div(<2 x half> %x, <2 x half
; GFX12-NEXT: v_pk_fma_f16 v0, v0, v1, v2
; GFX12-NEXT: global_store_b32 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2
-; GFX900-NEXT: global_store_dword v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2
-; GFX942-NEXT: global_store_dword v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %x, <2 x half> %y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <2 x half> %val, ptr addrspace(1) %out
ret void
@@ -258,6 +272,17 @@ define void @v_constained_fma_v3f16_fpexcept_strict_div(<3 x half> %x, <3 x half
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX9-NEXT: global_store_short v[6:7], v0, off
+; GFX9-NEXT: global_store_short_d16_hi v[6:7], v0, off offset:2
+; GFX9-NEXT: global_store_short v[6:7], v1, off offset:4
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -283,16 +308,6 @@ define void @v_constained_fma_v3f16_fpexcept_strict_div(<3 x half> %x, <3 x half
; GFX12-NEXT: global_store_d16_hi_b16 v[6:7], v0, off offset:2
; GFX12-NEXT: global_store_b16 v[6:7], v1, off offset:4
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX9-LABEL: v_constained_fma_v3f16_fpexcept_strict_div:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX9-NEXT: global_store_short v[6:7], v0, off
-; GFX9-NEXT: global_store_short_d16_hi v[6:7], v0, off offset:2
-; GFX9-NEXT: global_store_short v[6:7], v1, off offset:4
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
%val = call <3 x half> @llvm.experimental.constrained.fma.v3f16(<3 x half> %x, <3 x half> %y, <3 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <3 x half> %val, ptr addrspace(1) %out
ret void
@@ -302,32 +317,68 @@ define void @v_constained_fma_v4f16_fpexcept_strict_uni(<4 x half> inreg %x, <4
; GFX8-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s21, 16
-; GFX8-NEXT: s_lshr_b32 s5, s19, 16
-; GFX8-NEXT: s_lshr_b32 s6, s17, 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_fma_f16 v2, s6, v2, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s19
-; GFX8-NEXT: v_mov_b32_e32 v4, s21
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_fma_f16 v3, s17, v3, v4
-; GFX8-NEXT: s_lshr_b32 s4, s20, 16
-; GFX8-NEXT: s_lshr_b32 s5, s18, 16
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v2
-; GFX8-NEXT: s_lshr_b32 s6, s16, 16
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_fma_f16 v2, s6, v2, v4
-; GFX8-NEXT: v_mov_b32_e32 v4, s18
-; GFX8-NEXT: v_mov_b32_e32 v5, s20
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_fma_f16 v4, s16, v4, v5
-; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s18
+; GFX8-NEXT: v_mov_b32_e32 v3, s20
+; GFX8-NEXT: s_lshr_b32 s6, s18, 16
+; GFX8-NEXT: s_lshr_b32 s8, s20, 16
+; GFX8-NEXT: v_fma_f16 v2, s16, v2, v3
+; GFX8-NEXT: s_lshr_b32 s4, s16, 16
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s8
+; GFX8-NEXT: v_fma_f16 v2, s4, v2, v3
+; GFX8-NEXT: v_readfirstlane_b32 s4, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s19
+; GFX8-NEXT: v_mov_b32_e32 v3, s21
+; GFX8-NEXT: s_lshr_b32 s7, s19, 16
+; GFX8-NEXT: s_lshr_b32 s9, s21, 16
+; GFX8-NEXT: v_fma_f16 v2, s17, v2, v3
+; GFX8-NEXT: s_lshr_b32 s5, s17, 16
+; GFX8-NEXT: v_readfirstlane_b32 s6, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s7
+; GFX8-NEXT: v_mov_b32_e32 v3, s9
+; GFX8-NEXT: v_fma_f16 v2, s5, v2, v3
+; GFX8-NEXT: v_readfirstlane_b32 s5, v2
+; GFX8-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT: s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT: s_bfe_u32 s7, s10, 0x100000
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_bfe_u32 s6, s6, 0x100000
+; GFX8-NEXT: s_lshl_b32 s5, s5, 16
+; GFX8-NEXT: s_or_b32 s4, s7, s4
+; GFX8-NEXT: s_or_b32 s5, s6, s5
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s18
+; GFX900-NEXT: v_mov_b32_e32 v3, s20
+; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
+; GFX900-NEXT: v_mov_b32_e32 v3, s19
+; GFX900-NEXT: v_mov_b32_e32 v4, s21
+; GFX900-NEXT: v_pk_fma_f16 v3, s17, v3, v4
+; GFX900-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_mov_b32_e32 v3, s16
+; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
+; GFX942-NEXT: v_mov_b32_e32 v3, s3
+; GFX942-NEXT: v_mov_b32_e32 v4, s17
+; GFX942-NEXT: v_pk_fma_f16 v3, s1, v3, v4
+; GFX942-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -365,30 +416,6 @@ define void @v_constained_fma_v4f16_fpexcept_strict_uni(<4 x half> inreg %x, <4
; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v3, s1
; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s18
-; GFX900-NEXT: v_mov_b32_e32 v3, s20
-; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3
-; GFX900-NEXT: v_mov_b32_e32 v3, s19
-; GFX900-NEXT: v_mov_b32_e32 v4, s21
-; GFX900-NEXT: v_pk_fma_f16 v3, s17, v3, v4
-; GFX900-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_v4f16_fpexcept_strict_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s2
-; GFX942-NEXT: v_mov_b32_e32 v3, s16
-; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3
-; GFX942-NEXT: v_mov_b32_e32 v3, s3
-; GFX942-NEXT: v_mov_b32_e32 v4, s17
-; GFX942-NEXT: v_pk_fma_f16 v3, s1, v3, v4
-; GFX942-NEXT: global_store_dwordx2 v[0:1], v[2:3], off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <4 x half> %val, ptr addrspace(1) %out
ret void
@@ -398,24 +425,34 @@ define void @v_constained_fma_v4f16_fpexcept_strict_div(<4 x half> %x, <4 x half
; GFX8-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v5
-; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX8-NEXT: v_fma_f16 v8, v10, v9, v8
-; GFX8-NEXT: v_lshlrev_b32_e32 v8, 16, v8
-; GFX8-NEXT: v_fma_f16 v1, v1, v3, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v8
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v4
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GFX8-NEXT: v_fma_f16 v3, v8, v5, v3
-; GFX8-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; GFX8-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; GFX8-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX8-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; GFX8-NEXT: v_fma_f16 v0, v0, v2, v4
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX8-NEXT: v_fma_f16 v2, v8, v10, v12
+; GFX8-NEXT: v_mov_b32_e32 v4, 16
+; GFX8-NEXT: v_fma_f16 v1, v1, v3, v5
+; GFX8-NEXT: v_fma_f16 v3, v9, v11, v13
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_sdwa v2, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: flat_store_dwordx2 v[6:7], v[0:1]
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX9-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
+; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
+; GFX9-NEXT: global_store_dwordx2 v[6:7], v[0:1], off
+; GFX9-NEXT: s_waitcnt vmcnt(0)
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -435,14 +472,6 @@ define void @v_constained_fma_v4f16_fpexcept_strict_div(<4 x half> %x, <4 x half
; GFX12-NEXT: v_pk_fma_f16 v1, v1, v3, v5
; GFX12-NEXT: global_store_b64 v[6:7], v[0:1], off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX9-LABEL: v_constained_fma_v4f16_fpexcept_strict_div:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_pk_fma_f16 v0, v0, v2, v4
-; GFX9-NEXT: v_pk_fma_f16 v1, v1, v3, v5
-; GFX9-NEXT: global_store_dwordx2 v[6:7], v[0:1], off
-; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_setpc_b64 s[30:31]
%val = call <4 x half> @llvm.experimental.constrained.fma.v4f16(<4 x half> %x, <4 x half> %y, <4 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store <4 x half> %val, ptr addrspace(1) %out
ret void
@@ -459,30 +488,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT: v_mov_b16_e32 v2.l, s1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_fma_f16 v2.l, s0, v2.l, -s2
-; GFX11-NEXT: global_store_b16 v[0:1], v2, off
-; GFX11-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX12-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
-; GFX12: ; %bb.0:
-; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: s_wait_expcnt 0x0
-; GFX12-NEXT: s_wait_samplecnt 0x0
-; GFX12-NEXT: s_wait_bvhcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_fmac_f16 s2, s0, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
-; GFX12-NEXT: global_store_b16 v[0:1], v2, off
-; GFX12-NEXT: s_setpc_b64 s[30:31]
; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX900: ; %bb.0:
; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -492,6 +497,7 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX900-NEXT: global_store_short v[0:1], v2, off
; GFX900-NEXT: s_waitcnt vmcnt(0)
; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
; GFX942: ; %bb.0:
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -501,6 +507,31 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_uni(half inreg %x, half i
; GFX942-NEXT: global_store_short v[0:1], v2, off
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT: v_fma_f16 v2.l, s0, v2.l, -s2
+; GFX11-NEXT: global_store_b16 v[0:1], v2, off
+; GFX11-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_uni:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX12-NEXT: s_wait_expcnt 0x0
+; GFX12-NEXT: s_wait_samplecnt 0x0
+; GFX12-NEXT: s_wait_bvhcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_xor_b32 s2, s2, 0x8000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_fmac_f16 s2, s0, s1
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
+; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
+; GFX12-NEXT: global_store_b16 v[0:1], v2, off
+; GFX12-NEXT: s_setpc_b64 s[30:31]
%neg.z = fneg half %z
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %neg.z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
@@ -516,6 +547,24 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_div(half %x, half %y, hal
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, v0, v1, -v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -533,22 +582,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_div(half %x, half %y, hal
; GFX12-NEXT: v_fma_f16 v0.l, v0.l, v1.l, -v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, v0, v1, -v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, v0, v1, -v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.z = fneg half %z
%val = call half @llvm.experimental.constrained.fma.f16(half %x, half %y, half %neg.z, metadata !"round.tonearest", metadata !"fpexcept.strict")
store half %val, ptr addrspace(1) %out
@@ -566,6 +599,26 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni(half inreg %x, h
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_fma_f16 v2, -s16, -v2, v3
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_fma_f16 v2, -s0, -v2, v3
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -591,24 +644,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni(half inreg %x, h
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, -s16, -v2, v3
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, -s0, -v2, v3
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg half %x
%neg.y = fneg half %y
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -625,6 +660,24 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_div(half %x, half %y
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, -v0, -v1, v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, -v0, -v1, v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -642,22 +695,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fneg_fneg_div(half %x, half %y
; GFX12-NEXT: v_fma_f16 v0.l, -v0.l, -v1.l, v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, -v0, -v1, v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fneg_fneg_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, -v0, -v1, v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg half %x
%neg.y = fneg half %y
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -676,6 +713,26 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni(half inreg %x, h
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_fma_f16 v2, |s16|, |v2|, v3
+; GFX900-NEXT: global_store_short v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_fma_f16 v2, |s0|, |v2|, v3
+; GFX942-NEXT: global_store_short v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -701,24 +758,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni(half inreg %x, h
; GFX12-NEXT: v_mov_b16_e32 v2.l, s2
; GFX12-NEXT: global_store_b16 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_fma_f16 v2, |s16|, |v2|, v3
-; GFX900-NEXT: global_store_short v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_fma_f16 v2, |s0|, |v2|, v3
-; GFX942-NEXT: global_store_short v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = call half @llvm.fabs.f16(half %x) #0
%neg.y = call half @llvm.fabs.f16(half %y) #0
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -735,6 +774,24 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_div(half %x, half %y
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
+; GFX900-NEXT: global_store_short v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
+; GFX942-NEXT: global_store_short v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -752,22 +809,6 @@ define void @v_constained_fma_f16_fpexcept_strict_fabs_fabs_div(half %x, half %y
; GFX12-NEXT: v_fma_f16 v0.l, |v0.l|, |v1.l|, v2.l
; GFX12-NEXT: global_store_b16 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
-; GFX900-NEXT: global_store_short v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_f16_fpexcept_strict_fabs_fabs_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_fma_f16 v0, |v0|, |v1|, v2
-; GFX942-NEXT: global_store_short v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = call half @llvm.fabs.f16(half %x) #0
%neg.y = call half @llvm.fabs.f16(half %y) #0
%val = call half @llvm.experimental.constrained.fma.f16(half %neg.x, half %neg.y, half %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -779,21 +820,52 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni(<2 x half> inr
; GFX8-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s18, 16
-; GFX8-NEXT: s_lshr_b32 s5, s17, 16
-; GFX8-NEXT: s_lshr_b32 s6, s16, 16
+; GFX8-NEXT: v_mov_b32_e32 v2, s16
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT: v_readfirstlane_b32 s4, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s17
+; GFX8-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-NEXT: v_readfirstlane_b32 s5, v2
; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_fma_f16 v2, -s6, -v2, v3
-; GFX8-NEXT: v_mov_b32_e32 v3, s17
-; GFX8-NEXT: v_mov_b32_e32 v4, s18
-; GFX8-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX8-NEXT: v_fma_f16 v3, -s16, -v3, v4
-; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX8-NEXT: v_mov_b32_e32 v3, s18
+; GFX8-NEXT: s_lshr_b32 s7, s5, 16
+; GFX8-NEXT: s_lshr_b32 s8, s18, 16
+; GFX8-NEXT: v_fma_f16 v2, s4, v2, v3
+; GFX8-NEXT: s_lshr_b32 s6, s4, 16
+; GFX8-NEXT: v_readfirstlane_b32 s4, v2
+; GFX8-NEXT: v_mov_b32_e32 v2, s7
+; GFX8-NEXT: v_mov_b32_e32 v3, s8
+; GFX8-NEXT: v_fma_f16 v2, s6, v2, v3
+; GFX8-NEXT: v_readfirstlane_b32 s5, v2
+; GFX8-NEXT: s_bfe_u32 s5, s5, 0x100000
+; GFX8-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GFX8-NEXT: s_lshl_b32 s5, s5, 16
+; GFX8-NEXT: s_or_b32 s4, s4, s5
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_mov_b32_e32 v2, s17
+; GFX900-NEXT: v_mov_b32_e32 v3, s18
+; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX900-NEXT: global_store_dword v[0:1], v2, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v2, s1
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX942-NEXT: global_store_dword v[0:1], v2, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -828,24 +900,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni(<2 x half> inr
; GFX12-NEXT: v_mov_b32_e32 v2, s0
; GFX12-NEXT: global_store_b32 v[0:1], v2, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s17
-; GFX900-NEXT: v_mov_b32_e32 v3, s18
-; GFX900-NEXT: v_pk_fma_f16 v2, s16, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX900-NEXT: global_store_dword v[0:1], v2, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_uni:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s2
-; GFX942-NEXT: v_pk_fma_f16 v2, s0, v2, v3 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX942-NEXT: global_store_dword v[0:1], v2, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg <2 x half> %x
%neg.y = fneg <2 x half> %y
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %neg.x, <2 x half> %neg.y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
@@ -857,17 +911,38 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div(<2 x half> %x,
; GFX8-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX8-NEXT: v_fma_f16 v5, -v7, -v6, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX8-NEXT: v_fma_f16 v0, -v0, -v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX8-NEXT: v_fma_f16 v0, v0, v1, v2
+; GFX8-NEXT: v_fma_f16 v1, v5, v6, v7
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: flat_store_dword v[3:4], v0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
+; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
+; GFX900: ; %bb.0:
+; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX900-NEXT: global_store_dword v[3:4], v0, off
+; GFX900-NEXT: s_waitcnt vmcnt(0)
+; GFX900-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
+; GFX942: ; %bb.0:
+; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-NEXT: v_mov_b32_e32 v6, v3
+; GFX942-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
+; GFX942-NEXT: global_store_dword v[6:7], v0, off
+; GFX942-NEXT: s_waitcnt vmcnt(0)
+; GFX942-NEXT: s_setpc_b64 s[30:31]
+;
; GFX11-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -885,22 +960,6 @@ define void @v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div(<2 x half> %x,
; GFX12-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
; GFX12-NEXT: global_store_b32 v[3:4], v0, off
; GFX12-NEXT: s_setpc_b64 s[30:31]
-; GFX900-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
-; GFX900: ; %bb.0:
-; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX900-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX900-NEXT: global_store_dword v[3:4], v0, off
-; GFX900-NEXT: s_waitcnt vmcnt(0)
-; GFX900-NEXT: s_setpc_b64 s[30:31]
-; GFX942-LABEL: v_constained_fma_v2f16_fpexcept_strict_fneg_fneg_div:
-; GFX942: ; %bb.0:
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v6, v3
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_pk_fma_f16 v0, v0, v1, v2 neg_lo:[1,1,0] neg_hi:[1,1,0]
-; GFX942-NEXT: global_store_dword v[6:7], v0, off
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
%neg.x = fneg <2 x half> %x
%neg.y = fneg <2 x half> %y
%val = call <2 x half> @llvm.experimental.constrained.fma.v2f16(<2 x half> %neg.x, <2 x half> %neg.y, <2 x half> %z, metadata !"round.tonearest", metadata !"fpexcept.strict")
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
index de4d43ee67070..a910fb96a8d9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01-amd-amdpal < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgpu12.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX12 %s
define i16 @s_sub_i16(i16 inreg %a, i16 inreg %b) {
; GFX7-LABEL: s_sub_i16:
@@ -203,13 +203,14 @@ define <2 x i16> @v_sub_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX7-LABEL: v_sub_v2i16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_sub_i32_e32 v2, vcc, v3, v2
+; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_sub_i32_e32 v1, vcc, v2, v3
+; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_sub_v2i16:
@@ -221,9 +222,9 @@ define <2 x i16> @v_sub_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX8-LABEL: v_sub_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_sub_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sub_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
index 973717553c89d..195a3a623f0d6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sub.v2i16.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefix=GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal < %s | FileCheck -check-prefix=GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefix=GFX11 %s
@@ -14,9 +14,9 @@ define <2 x i16> @v_sub_v2i16(<2 x i16> %a, <2 x i16> %b) {
; GFX8-LABEL: v_sub_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_sub_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sub_v2i16:
@@ -45,9 +45,9 @@ define <2 x i16> @v_sub_v2i16_fneg_lhs(<2 x half> %a, <2 x i16> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX8-NEXT: v_sub_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_sub_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sub_v2i16_fneg_lhs:
@@ -78,9 +78,9 @@ define <2 x i16> @v_sub_v2i16_fneg_rhs(<2 x i16> %a, <2 x half> %b) {
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
-; GFX8-NEXT: v_sub_u16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_sub_u16_e32 v2, v0, v1
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_sub_v2i16_fneg_rhs:
@@ -243,11 +243,13 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_splat(<2 x i16> inreg %a) {
;
; GFX8-LABEL: s_sub_v2i16_neg_inline_imm_splat:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_add_i32 s1, s1, 64
; GFX8-NEXT: s_add_i32 s0, s0, 64
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_add_i32 s0, s0, 0x400000
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16_neg_inline_imm_splat:
@@ -281,11 +283,13 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_lo(<2 x i16> inreg %a) {
;
; GFX8-LABEL: s_sub_v2i16_neg_inline_imm_lo:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_add_i32 s1, s1, 0xfffc
; GFX8-NEXT: s_add_i32 s0, s0, 64
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_add_i32 s0, s0, 0xfffc0000
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16_neg_inline_imm_lo:
@@ -319,11 +323,13 @@ define amdgpu_ps i32 @s_sub_v2i16_neg_inline_imm_hi(<2 x i16> inreg %a) {
;
; GFX8-LABEL: s_sub_v2i16_neg_inline_imm_hi:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s1, s0, 0xffff0000
+; GFX8-NEXT: s_lshr_b32 s1, s0, 16
+; GFX8-NEXT: s_add_i32 s1, s1, 64
; GFX8-NEXT: s_add_i32 s0, s0, 0xfffc
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s1, s0
-; GFX8-NEXT: s_add_i32 s0, s0, 0x400000
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16_neg_inline_imm_hi:
@@ -358,13 +364,14 @@ define amdgpu_ps i32 @s_sub_v2i16(<2 x i16> inreg %a, <2 x i16> inreg %b) {
;
; GFX8-LABEL: s_sub_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_sub_i32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_sub_i32 s1, s2, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16:
@@ -404,14 +411,17 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs(<2 x half> inreg %a, <2 x i16> inreg
;
; GFX8-LABEL: s_sub_v2i16_fneg_lhs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: s_sub_i32 s2, s3, s2
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_sub_i32 s1, s2, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16_fneg_lhs:
@@ -457,14 +467,17 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_rhs(<2 x i16> inreg %a, <2 x half> inreg
;
; GFX8-LABEL: s_sub_v2i16_fneg_rhs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x80008000
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
; GFX8-NEXT: s_lshr_b32 s2, s0, 16
; GFX8-NEXT: s_lshr_b32 s3, s1, 16
-; GFX8-NEXT: s_sub_i32 s2, s2, s3
; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_sub_i32 s1, s2, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16_fneg_rhs:
@@ -513,15 +526,20 @@ define amdgpu_ps i32 @s_sub_v2i16_fneg_lhs_fneg_rhs(<2 x half> inreg %a, <2 x ha
;
; GFX8-LABEL: s_sub_v2i16_fneg_lhs_fneg_rhs:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_xor_b32 s0, s0, 0x80008000
-; GFX8-NEXT: s_xor_b32 s1, s1, 0x80008000
-; GFX8-NEXT: s_sub_i32 s2, s0, s1
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s2, s0
+; GFX8-NEXT: s_sub_i32 s1, s2, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_sub_v2i16_fneg_lhs_fneg_rhs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 8bf28a0fb55ba..54f7339ab4c32 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
define i7 @v_uaddsat_i7(i7 %lhs, i7 %rhs) {
; GFX6-LABEL: v_uaddsat_i7:
@@ -534,149 +534,198 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX6-LABEL: v_uaddsat_v4i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v1
-; GFX6-NEXT: v_and_b32_e32 v3, 0xff, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 24, v0
-; GFX6-NEXT: v_bfe_u32 v6, v1, 8, 8
-; GFX6-NEXT: v_bfe_u32 v7, v0, 8, 8
-; GFX6-NEXT: v_bfe_u32 v1, v1, 16, 8
-; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 8
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_not_b32_e32 v8, v0
+; GFX6-NEXT: v_min_u32_e32 v1, v8, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, v3, v2
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, v7, v6
-; GFX6-NEXT: v_min_u32_e32 v4, 0xff, v4
-; GFX6-NEXT: v_min_u32_e32 v0, 0xff, v0
-; GFX6-NEXT: v_min_u32_e32 v2, 0xff, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_min_u32_e32 v1, 0xff, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT: v_not_b32_e32 v5, v1
+; GFX6-NEXT: v_min_u32_e32 v2, v5, v2
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT: v_not_b32_e32 v5, v2
+; GFX6-NEXT: v_min_u32_e32 v3, v5, v3
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT: v_not_b32_e32 v5, v3
+; GFX6-NEXT: v_min_u32_e32 v4, v5, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 24
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_uaddsat_v4i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX8-NEXT: v_and_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_sdwa v4, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_add_u16_sdwa v5, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_add_u16_sdwa v6, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX8-NEXT: v_add_u16_e32 v1, v4, v3
-; GFX8-NEXT: v_min_u16_e32 v5, 0xff, v5
-; GFX8-NEXT: v_min_u16_sdwa v6, v6, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_min_u16_e32 v1, 0xff, v1
-; GFX8-NEXT: v_or_b32_e32 v5, v5, v6
-; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, 8
+; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT: v_add_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT: v_add_u16_e64 v1, v3, v2 clamp
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v6
+; GFX8-NEXT: v_add_u16_e64 v2, v2, v3 clamp
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v5
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT: v_add_u16_e64 v3, v3, v4 clamp
+; GFX8-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT: v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_sdwa v1, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_sdwa v1, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0xff
-; GFX9-NEXT: v_and_b32_sdwa v2, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_add_u16_sdwa v4, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NEXT: v_add_u16_e32 v2, v3, v2
-; GFX9-NEXT: v_add_u16_sdwa v3, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NEXT: v_min_u16_e32 v3, 0xff, v3
-; GFX9-NEXT: v_min_u16_sdwa v0, v0, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_min_u16_sdwa v4, v4, s4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-NEXT: v_min_u16_e32 v2, 0xff, v2
-; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
+; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
+; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 24
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_uaddsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX10-NEXT: v_mov_b32_e32 v4, 0xff
-; GFX10-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v7, 0xff, v1
-; GFX10-NEXT: v_and_b32_e32 v8, 0xff, v0
-; GFX10-NEXT: v_add_nc_u16 v2, v3, v2
-; GFX10-NEXT: v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_add_nc_u16 v3, v6, v5
-; GFX10-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_add_nc_u16 v4, v8, v7
-; GFX10-NEXT: v_min_u16 v2, 0xff, v2
-; GFX10-NEXT: v_min_u16 v3, 0xff, v3
-; GFX10-NEXT: v_add_nc_u16 v0, v0, v1
-; GFX10-NEXT: v_min_u16 v1, 0xff, v4
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_min_u16 v0, 0xff, v0
-; GFX10-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
+; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-NEXT: v_pk_lshrrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_uaddsat_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, 8, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v1.h, v2.h, v1.h
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v2.h, v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_add_nc_u16 v0.h, v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_min_u16 v1.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_min_u16 v1.h, 0xff, v2.h
-; GFX11-TRUE16-NEXT: v_min_u16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_min_u16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.l
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v1.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v6 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v1, v2, v1 clamp
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_uaddsat_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v1
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v2, v3, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v5
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v1, v6, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v7
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v0, v0, v4
-; GFX11-FAKE16-NEXT: v_min_u16 v2, 0xff, v2
-; GFX11-FAKE16-NEXT: v_min_u16 v1, 0xff, v1
-; GFX11-FAKE16-NEXT: v_add_nc_u16 v3, v5, v3
-; GFX11-FAKE16-NEXT: v_min_u16 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_min_u16 v3, 0xff, v3
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v3, 16, v5
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v6, v0, 16
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v2, v2, v3 clamp
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xff, v1, v2
+; GFX11-FAKE16-NEXT: v_or3_b32 v0, v1, v3, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -688,113 +737,303 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
define amdgpu_ps i32 @s_uaddsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
; GFX6-LABEL: s_uaddsat_v4i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s2, s1, 0xff
-; GFX6-NEXT: s_and_b32 s3, s0, 0xff
-; GFX6-NEXT: s_lshr_b32 s4, s1, 24
-; GFX6-NEXT: s_lshr_b32 s5, s0, 24
-; GFX6-NEXT: s_bfe_u32 s6, s1, 0x80008
-; GFX6-NEXT: s_bfe_u32 s7, s0, 0x80008
-; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX6-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX6-NEXT: s_add_i32 s5, s5, s4
+; GFX6-NEXT: s_lshr_b32 s2, s0, 8
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_lshr_b32 s4, s0, 24
+; GFX6-NEXT: s_lshl_b32 s0, s0, 24
+; GFX6-NEXT: s_lshr_b32 s5, s1, 8
+; GFX6-NEXT: s_lshr_b32 s6, s1, 16
+; GFX6-NEXT: s_lshr_b32 s7, s1, 24
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_not_b32 s8, s0
+; GFX6-NEXT: s_min_u32 s1, s8, s1
; GFX6-NEXT: s_add_i32 s0, s0, s1
-; GFX6-NEXT: s_add_i32 s7, s7, s6
-; GFX6-NEXT: s_min_u32 s4, s5, 0xff
-; GFX6-NEXT: s_min_u32 s0, s0, 0xff
-; GFX6-NEXT: s_add_i32 s3, s3, s2
-; GFX6-NEXT: s_min_u32 s2, s7, 0xff
-; GFX6-NEXT: s_lshl_b32 s4, s4, 24
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_min_u32 s1, s3, 0xff
-; GFX6-NEXT: s_lshl_b32 s2, s2, 8
-; GFX6-NEXT: s_or_b32 s0, s4, s0
-; GFX6-NEXT: s_or_b32 s1, s1, s2
-; GFX6-NEXT: s_or_b32 s0, s1, s0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 24
+; GFX6-NEXT: s_lshl_b32 s2, s5, 24
+; GFX6-NEXT: s_not_b32 s5, s1
+; GFX6-NEXT: s_min_u32 s2, s5, s2
+; GFX6-NEXT: s_add_i32 s1, s1, s2
+; GFX6-NEXT: s_lshl_b32 s2, s3, 24
+; GFX6-NEXT: s_lshl_b32 s3, s6, 24
+; GFX6-NEXT: s_not_b32 s5, s2
+; GFX6-NEXT: s_min_u32 s3, s5, s3
+; GFX6-NEXT: s_add_i32 s2, s2, s3
+; GFX6-NEXT: s_lshl_b32 s3, s4, 24
+; GFX6-NEXT: s_lshl_b32 s4, s7, 24
+; GFX6-NEXT: s_not_b32 s5, s3
+; GFX6-NEXT: s_lshr_b32 s1, s1, 24
+; GFX6-NEXT: s_min_u32 s4, s5, s4
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: s_lshr_b32 s2, s2, 24
+; GFX6-NEXT: s_add_i32 s3, s3, s4
+; GFX6-NEXT: v_alignbit_b32 v0, s1, v0, 24
+; GFX6-NEXT: s_lshr_b32 s3, s3, 24
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_lshl_b32 s1, s3, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_uaddsat_v4i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 24
-; GFX8-NEXT: s_lshr_b32 s3, s0, 24
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x80008
-; GFX8-NEXT: s_add_i32 s3, s3, s2
-; GFX8-NEXT: s_min_u32 s2, s3, 0xff
-; GFX8-NEXT: s_bfe_u32 s3, s1, 0x80010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x80010
-; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_add_i32 s5, s5, s4
-; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: s_and_b32 s1, s5, 0x1ff
-; GFX8-NEXT: s_add_i32 s6, s6, s3
-; GFX8-NEXT: s_min_u32 s1, s1, 0xff
-; GFX8-NEXT: s_lshl_b32 s2, s2, 8
-; GFX8-NEXT: s_min_u32 s3, s6, 0xff
-; GFX8-NEXT: s_min_u32 s0, s0, 0xff
+; GFX8-NEXT: s_lshr_b32 s5, s1, 8
+; GFX8-NEXT: s_lshr_b32 s6, s1, 16
+; GFX8-NEXT: s_lshr_b32 s7, s1, 24
; GFX8-NEXT: s_lshl_b32 s1, s1, 8
-; GFX8-NEXT: s_or_b32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: s_lshr_b32 s4, s0, 24
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s5, 8
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_add_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: s_lshl_b32 s3, s6, 8
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_add_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s3, s4, 8
+; GFX8-NEXT: s_lshl_b32 s4, s7, 8
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: v_add_u16_e64 v0, s3, v0 clamp
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s1, s1, 8
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
+; GFX8-NEXT: s_lshr_b32 s2, s2, 8
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s3, s3, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s3, 24
; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v4i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x80008
-; GFX9-NEXT: s_lshr_b32 s2, s1, 24
-; GFX9-NEXT: s_lshr_b32 s3, s0, 24
-; GFX9-NEXT: s_add_i32 s5, s5, s4
-; GFX9-NEXT: s_and_b32 s6, s1, 0xff
-; GFX9-NEXT: s_and_b32 s7, s0, 0xff
-; GFX9-NEXT: s_and_b32 s4, s5, 0x1ff
-; GFX9-NEXT: s_add_i32 s3, s3, s2
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX9-NEXT: s_add_i32 s7, s7, s6
-; GFX9-NEXT: s_min_u32 s4, s4, 0xff
-; GFX9-NEXT: s_min_u32 s2, s3, 0xff
-; GFX9-NEXT: s_add_i32 s0, s0, s1
-; GFX9-NEXT: s_min_u32 s6, s7, 0xff
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-NEXT: s_lshr_b32 s4, s0, 24
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
; GFX9-NEXT: s_lshl_b32 s4, s4, 8
-; GFX9-NEXT: s_lshl_b32 s2, s2, 8
-; GFX9-NEXT: s_min_u32 s0, s0, 0xff
-; GFX9-NEXT: s_or_b32 s4, s6, s4
+; GFX9-NEXT: s_lshr_b32 s5, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshr_b32 s6, s1, 16
+; GFX9-NEXT: s_lshr_b32 s7, s1, 24
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s6, s7
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT: v_pk_add_u16 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_pk_add_u16 v0, s2, v0 clamp
+; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshr_b32 s1, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT: s_and_b32 s2, s0, 0xff
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
; GFX9-NEXT: s_or_b32 s0, s0, s2
-; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
-; GFX9-NEXT: s_lshl_b32 s0, s0, 16
-; GFX9-NEXT: s_or_b32 s0, s4, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_uaddsat_v4i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX10PLUS-NEXT: s_bfe_u32 s5, s0, 0x80008
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s1, 24
-; GFX10PLUS-NEXT: s_lshr_b32 s3, s0, 24
-; GFX10PLUS-NEXT: s_add_i32 s5, s5, s4
-; GFX10PLUS-NEXT: s_and_b32 s6, s1, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s4, s0, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s5, s5, 0x1ff
-; GFX10PLUS-NEXT: s_add_i32 s3, s3, s2
-; GFX10PLUS-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX10PLUS-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX10PLUS-NEXT: s_add_i32 s4, s4, s6
-; GFX10PLUS-NEXT: s_min_u32 s5, s5, 0xff
-; GFX10PLUS-NEXT: s_min_u32 s2, s3, 0xff
-; GFX10PLUS-NEXT: s_add_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_min_u32 s4, s4, 0xff
-; GFX10PLUS-NEXT: s_lshl_b32 s5, s5, 8
-; GFX10PLUS-NEXT: s_lshl_b32 s1, s2, 8
-; GFX10PLUS-NEXT: s_min_u32 s0, s0, 0xff
-; GFX10PLUS-NEXT: s_or_b32 s2, s4, s5
-; GFX10PLUS-NEXT: s_or_b32 s0, s0, s1
-; GFX10PLUS-NEXT: s_and_b32 s1, s2, 0xffff
-; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 16
-; GFX10PLUS-NEXT: s_or_b32 s0, s1, s0
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_uaddsat_v4i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10-NEXT: s_lshr_b32 s5, s1, 8
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s4, s0, 24
+; GFX10-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10-NEXT: s_lshr_b32 s7, s1, 24
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s4, s6, s7
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s3, s2, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_lshr_b32 s5, s4, 16
+; GFX10-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_lshl_b32 s4, s4, 0x80008
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: v_pk_add_u16 v0, s0, s1 clamp
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s4, s5
+; GFX10-NEXT: v_pk_add_u16 v1, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_lshr_b32 s3, s3, 0x80008
+; GFX10-NEXT: s_lshr_b32 s1, s1, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_and_b32 s3, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, 16
+; GFX10-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_uaddsat_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, s2, s3 clamp
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: v_pk_add_u16 v1, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 0x80008
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 24
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_uaddsat_v4i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v0, s2, s3 clamp
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: v_pk_add_u16 v1, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 0x80008
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s3, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 24
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
%result = call <4 x i8> @llvm.uadd.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
@@ -1957,9 +2196,9 @@ define <2 x i16> @v_uaddsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX8-LABEL: v_uaddsat_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v2, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v0, v0, v1 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_add_u16_e64 v2, v0, v1 clamp
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddsat_v2i16:
@@ -2000,15 +2239,18 @@ define amdgpu_ps i32 @s_uaddsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
;
; GFX8-LABEL: s_uaddsat_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_u16_e64 v1, s0, v1 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_add_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v2i16:
@@ -2050,10 +2292,10 @@ define amdgpu_ps float @uaddsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
; GFX8-LABEL: uaddsat_v2i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_u16_sdwa v1, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u16_e64 v1, s0, v0 clamp
+; GFX8-NEXT: v_add_u16_sdwa v0, v2, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: uaddsat_v2i16_sv:
@@ -2092,10 +2334,10 @@ define amdgpu_ps float @uaddsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
; GFX8-LABEL: uaddsat_v2i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_u16_sdwa v1, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_add_u16_e64 v0, v0, s0 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_add_u16_e64 v1, v0, s0 clamp
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: uaddsat_v2i16_vs:
@@ -2160,12 +2402,12 @@ define <2 x float> @v_uaddsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
; GFX8-LABEL: v_uaddsat_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v4, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v0, v0, v2 clamp
-; GFX8-NEXT: v_add_u16_sdwa v2, v1, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v1, v1, v3 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_add_u16_e64 v4, v0, v2 clamp
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e64 v2, v1, v3 clamp
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddsat_v4i16:
@@ -2225,24 +2467,30 @@ define amdgpu_ps <2 x i32> @s_uaddsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
;
; GFX8-LABEL: s_uaddsat_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s4, s3, 16
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s4, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_add_u16_e64 v0, s4, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s1, v0 clamp
; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_u16_e64 v1, s1, v1 clamp
-; GFX8-NEXT: s_lshr_b32 s1, s2, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_add_u16_sdwa v1, v2, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_add_u16_e64 v2, s0, v2 clamp
-; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX8-NEXT: v_readfirstlane_b32 s0, v1
; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_add_u16_e64 v0, s5, v0 clamp
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v4i16:
@@ -2329,15 +2577,15 @@ define <3 x float> @v_uaddsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
; GFX8-LABEL: v_uaddsat_v6i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v6, v0, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v0, v0, v3 clamp
-; GFX8-NEXT: v_add_u16_sdwa v3, v1, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v1, v1, v4 clamp
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_add_u16_sdwa v3, v2, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v2, v2, v5 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v6
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_add_u16_e64 v6, v0, v3 clamp
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e64 v3, v1, v4 clamp
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e64 v4, v2, v5 clamp
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddsat_v6i16:
@@ -2415,33 +2663,42 @@ define amdgpu_ps <3 x i32> @s_uaddsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
;
; GFX8-LABEL: s_uaddsat_v6i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s6, s5, 16
-; GFX8-NEXT: s_lshr_b32 s7, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
-; GFX8-NEXT: v_add_u16_e64 v1, s2, v1 clamp
-; GFX8-NEXT: s_lshr_b32 s2, s4, 16
-; GFX8-NEXT: s_lshr_b32 s5, s1, 16
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s5
-; GFX8-NEXT: v_add_u16_sdwa v1, v2, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_add_u16_e64 v2, s1, v2 clamp
-; GFX8-NEXT: s_lshr_b32 s1, s3, 16
-; GFX8-NEXT: s_lshr_b32 s2, s0, 16
-; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_add_u16_sdwa v2, v3, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_add_u16_e64 v3, s0, v3 clamp
-; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX8-NEXT: v_readfirstlane_b32 s0, v2
-; GFX8-NEXT: v_readfirstlane_b32 s1, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: s_lshr_b32 s9, s3, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s9
+; GFX8-NEXT: v_add_u16_e64 v0, s6, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_lshr_b32 s10, s4, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_add_u16_e64 v0, s7, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
+; GFX8-NEXT: s_lshr_b32 s11, s5, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshr_b32 s8, s2, 16
; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s11
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s8, v0 clamp
+; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s4
+; GFX8-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s5
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v6i16:
@@ -2536,18 +2793,18 @@ define <4 x float> @v_uaddsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
; GFX8-LABEL: v_uaddsat_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_sdwa v8, v0, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v0, v0, v4 clamp
-; GFX8-NEXT: v_add_u16_sdwa v4, v1, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v1, v1, v5 clamp
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: v_add_u16_sdwa v4, v2, v6 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v2, v2, v6 clamp
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_add_u16_sdwa v4, v3, v7 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_u16_e64 v3, v3, v7 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_add_u16_e64 v8, v0, v4 clamp
+; GFX8-NEXT: v_add_u16_sdwa v0, v0, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e64 v4, v1, v5 clamp
+; GFX8-NEXT: v_add_u16_sdwa v1, v1, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e64 v5, v2, v6 clamp
+; GFX8-NEXT: v_add_u16_sdwa v2, v2, v6 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_add_u16_e64 v6, v3, v7 clamp
+; GFX8-NEXT: v_add_u16_sdwa v3, v3, v7 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_uaddsat_v8i16:
@@ -2643,42 +2900,54 @@ define amdgpu_ps <4 x i32> @s_uaddsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
;
; GFX8-LABEL: s_uaddsat_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s8, s7, 16
-; GFX8-NEXT: s_lshr_b32 s9, s3, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s9
-; GFX8-NEXT: v_add_u16_sdwa v0, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
-; GFX8-NEXT: v_add_u16_e64 v1, s3, v1 clamp
-; GFX8-NEXT: s_lshr_b32 s3, s6, 16
-; GFX8-NEXT: s_lshr_b32 s7, s2, 16
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mov_b32_e32 v2, s7
-; GFX8-NEXT: v_add_u16_sdwa v1, v2, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_add_u16_e64 v2, s2, v2 clamp
-; GFX8-NEXT: s_lshr_b32 s2, s5, 16
-; GFX8-NEXT: s_lshr_b32 s3, s1, 16
-; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_add_u16_sdwa v2, v3, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: v_add_u16_e64 v3, s1, v3 clamp
-; GFX8-NEXT: s_lshr_b32 s1, s4, 16
-; GFX8-NEXT: s_lshr_b32 s2, s0, 16
-; GFX8-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NEXT: v_add_u16_sdwa v3, v4, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_add_u16_e64 v4, s0, v4 clamp
-; GFX8-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX8-NEXT: v_readfirstlane_b32 s0, v3
-; GFX8-NEXT: v_readfirstlane_b32 s1, v2
-; GFX8-NEXT: v_readfirstlane_b32 s2, v1
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_add_u16_e64 v0, s8, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s13
+; GFX8-NEXT: v_add_u16_e64 v0, s9, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
+; GFX8-NEXT: v_add_u16_e64 v0, s10, v0 clamp
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: v_readfirstlane_b32 s6, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s3, v0 clamp
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s5
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s15
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: v_add_u16_e64 v0, s11, v0 clamp
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_uaddsat_v8i16:
@@ -2717,16 +2986,18 @@ define i48 @v_uaddsat_i48(i48 %lhs, i48 %rhs) {
; GFX6-LABEL: v_uaddsat_i48:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2
-; GFX6-NEXT: s_mov_b32 s4, -1
-; GFX6-NEXT: s_mov_b32 s5, 0xffff
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; GFX6-NEXT: v_cmp_gt_u64_e32 vcc, s[4:5], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v1, v2
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_uaddsat_i48:
@@ -2771,16 +3042,21 @@ define i48 @v_uaddsat_i48(i48 %lhs, i48 %rhs) {
define amdgpu_ps i48 @s_uaddsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
; GFX6-LABEL: s_uaddsat_i48:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: s_add_u32 s0, s0, s2
-; GFX6-NEXT: s_addc_u32 s1, s1, s3
-; GFX6-NEXT: v_mov_b32_e32 v0, -1
-; GFX6-NEXT: v_mov_b32_e32 v1, 0xffff
-; GFX6-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[0:1]
-; GFX6-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX6-NEXT: s_cselect_b32 s1, s1, 0xffff
-; GFX6-NEXT: s_cselect_b32 s0, s0, -1
+; GFX6-NEXT: s_cselect_b32 s2, 1, 0
+; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
+; GFX6-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-NEXT: s_addc_u32 s2, s1, s3
+; GFX6-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX6-NEXT: s_cmp_lg_u32 s2, s1
+; GFX6-NEXT: s_cselect_b32 s2, 1, 0
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s3
+; GFX6-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], -1, s[0:1]
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_uaddsat_i48:
@@ -2823,13 +3099,16 @@ define amdgpu_ps <2 x float> @uaddsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: s_mov_b32 s3, 0xffff
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v2, v1, vcc
-; GFX6-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v1, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_i48_sv:
@@ -2879,13 +3158,16 @@ define amdgpu_ps <2 x float> @uaddsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: s_mov_b32 s3, 0xffff
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc
-; GFX6-NEXT: v_cmp_gt_u64_e32 vcc, s[2:3], v[0:1]
-; GFX6-NEXT: v_mov_b32_e32 v2, 0xffff
-; GFX6-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v1, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, -1, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, -1, vcc
+; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: uaddsat_i48_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index 0998ce5159254..2ec45c8f5e7ff 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -1,5 +1,4 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; UNSUPPORTED: true
; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu8.02-amd-amdhsa < %s | FileCheck --check-prefix=GFX8 %s
; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefix=GFX9 %s
; RUN: llc -global-isel -amdgpu-codegenprepare-disable-idiv-expansion=1 -amdgpu-bypass-slow-div=0 -mtriple=amdgpu10.10-amd-amdhsa < %s | FileCheck -check-prefix=GFX10 %s
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll
index 913589f428134..d947340503e0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/umed3.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.02-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.02-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX8 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-mesa3d < %s | FileCheck -check-prefixes=GFX89,GFX9 %s
; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-mesa3d < %s | FileCheck -check-prefix=GFX10 %s
@@ -143,13 +143,13 @@ define <2 x i16> @test_max_K0min_K1Val__v2u16(<2 x i16> %a) {
; GFX8-LABEL: test_max_K0min_K1Val__v2u16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 17
-; GFX8-NEXT: v_min_u16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_min_u16_e32 v0, 17, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, 17
+; GFX8-NEXT: v_min_u16_e32 v1, 17, v0
+; GFX8-NEXT: v_min_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GFX8-NEXT: v_mov_b32_e32 v2, 12
-; GFX8-NEXT: v_max_u16_e32 v0, 12, v0
-; GFX8-NEXT: v_max_u16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_max_u16_e32 v1, 12, v1
+; GFX8-NEXT: v_max_u16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_max_K0min_K1Val__v2u16:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index b97e76373e7f5..fc4b7bd4c9cfd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -1,10 +1,10 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03-amd-amdpal -o - %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00-amd-amdpal -o - %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10-amd-amdpal -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=+real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00-amd-amdpal -mattr=-real-true16 -amdgpu-enable-delay-alu=0 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
define i7 @v_usubsat_i7(i7 %lhs, i7 %rhs) {
; GFX6-LABEL: v_usubsat_i7:
@@ -526,129 +526,194 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX6-LABEL: v_usubsat_v4i8:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_and_b32_e32 v2, 0xff, v1
-; GFX6-NEXT: v_and_b32_e32 v3, 0xff, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 24, v0
-; GFX6-NEXT: v_bfe_u32 v6, v1, 8, 8
-; GFX6-NEXT: v_bfe_u32 v7, v0, 8, 8
-; GFX6-NEXT: v_bfe_u32 v1, v1, 16, 8
-; GFX6-NEXT: v_bfe_u32 v0, v0, 16, 8
-; GFX6-NEXT: v_max_u32_e32 v0, v0, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v4, 24, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_min_u32_e32 v1, v0, v1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_max_u32_e32 v1, v3, v2
-; GFX6-NEXT: v_max_u32_e32 v5, v5, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v5
+; GFX6-NEXT: v_min_u32_e32 v2, v1, v2
; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2
-; GFX6-NEXT: v_max_u32_e32 v2, v7, v6
-; GFX6-NEXT: v_sub_i32_e32 v4, vcc, v5, v4
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v6
+; GFX6-NEXT: v_min_u32_e32 v3, v2, v3
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, v2, v3
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 24, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 24, v7
+; GFX6-NEXT: v_min_u32_e32 v4, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v1, 24, v1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 24, v2
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v4
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 24, v3
+; GFX6-NEXT: v_alignbit_b32 v0, v1, v0, 24
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_lshlrev_b32_e32 v1, 24, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubsat_v4i8:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, 0xff
-; GFX8-NEXT: v_and_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_and_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_sub_u16_sdwa v4, v0, v1 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX8-NEXT: v_sub_u16_sdwa v5, v0, v1 clamp dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 clamp dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX8-NEXT: v_sub_u16_e64 v1, v2, v3 clamp
-; GFX8-NEXT: v_or_b32_e32 v4, v4, v5
-; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_mov_b32_e32 v2, 8
+; GFX8-NEXT: v_lshrrev_b32_sdwa v3, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX8-NEXT: v_lshrrev_b32_sdwa v2, v2, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX8-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX8-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GFX8-NEXT: v_lshlrev_b16_e32 v1, 8, v1
+; GFX8-NEXT: v_sub_u16_e64 v0, v0, v1 clamp
+; GFX8-NEXT: v_sub_u16_e64 v1, v3, v2 clamp
+; GFX8-NEXT: v_lshlrev_b16_e32 v2, 8, v4
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v6
+; GFX8-NEXT: v_sub_u16_e64 v2, v2, v3 clamp
+; GFX8-NEXT: v_lshlrev_b16_e32 v3, 8, v5
+; GFX8-NEXT: v_lshlrev_b16_e32 v4, 8, v7
+; GFX8-NEXT: v_sub_u16_e64 v3, v3, v4 clamp
+; GFX8-NEXT: v_mov_b32_e32 v4, 0xff
+; GFX8-NEXT: v_and_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_and_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_sdwa v1, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_and_b32_sdwa v1, v3, v4 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubsat_v4i8:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_movk_i32 s4, 0xff
-; GFX9-NEXT: v_and_b32_sdwa v2, v1, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_and_b32_sdwa v3, v0, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT: v_sub_u16_sdwa v4, v0, v1 clamp dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
-; GFX9-NEXT: v_sub_u16_e64 v2, v3, v2 clamp
-; GFX9-NEXT: v_sub_u16_sdwa v3, v0, v1 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NEXT: v_sub_u16_sdwa v0, v0, v1 clamp dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
-; GFX9-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX9-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX9-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX9-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v5, 24, v1
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, 16, v6
+; GFX9-NEXT: v_lshl_or_b32 v3, v4, 16, v3
+; GFX9-NEXT: v_alignbit_b32 v1, v5, v1, 16
+; GFX9-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX9-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX9-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX9-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v3, 8
+; GFX9-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX9-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX9-NEXT: v_lshlrev_b32_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX9-NEXT: v_mov_b32_e32 v3, 24
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX9-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_usubsat_v4i8:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX10-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX10-NEXT: v_and_b32_e32 v4, 0xff, v1
-; GFX10-NEXT: v_mov_b32_e32 v5, 0xff
-; GFX10-NEXT: v_lshrrev_b32_e32 v6, 24, v1
-; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX10-NEXT: v_and_b32_e32 v8, 0xff, v0
-; GFX10-NEXT: v_sub_nc_u16 v2, v3, v2 clamp
-; GFX10-NEXT: v_and_b32_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_and_b32_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-NEXT: v_sub_nc_u16 v3, v7, v6 clamp
-; GFX10-NEXT: v_sub_nc_u16 v4, v8, v4 clamp
-; GFX10-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX10-NEXT: v_sub_nc_u16 v0, v0, v1 clamp
-; GFX10-NEXT: v_lshlrev_b16 v3, 8, v3
-; GFX10-NEXT: v_or_b32_e32 v1, v4, v2
-; GFX10-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX10-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX10-NEXT: v_lshl_or_b32 v0, v0, 16, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v3, 24, v0
+; GFX10-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX10-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; GFX10-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; GFX10-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX10-NEXT: v_alignbit_b32 v0, v3, v0, 16
+; GFX10-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX10-NEXT: v_mov_b32_e32 v4, 24
+; GFX10-NEXT: v_lshl_or_b32 v3, v5, 16, v6
+; GFX10-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX10-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX10-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX10-NEXT: v_mov_b32_e32 v1, 8
+; GFX10-NEXT: v_pk_lshrrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX10-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX10-NEXT: v_lshlrev_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX10-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX10-NEXT: v_and_or_b32 v1, 0xff, v2, v1
+; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX10-NEXT: v_or3_b32 v0, v1, v2, v0
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-TRUE16-LABEL: v_usubsat_v4i8:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_and_b16 v2.l, 0xff, v1.h
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v1.h, 8, v1.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b16 v2.h, 8, v0.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 24, v0
-; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v0.h
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v1.h, v2.h, v1.h clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v2.h, v4.l, v3.l clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.l, v0.l, v1.l clamp
-; GFX11-TRUE16-NEXT: v_sub_nc_u16 v0.h, v0.h, v2.l clamp
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.l, 8, v1.h
-; GFX11-TRUE16-NEXT: v_lshlrev_b16 v1.h, 8, v2.h
-; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v1.l
-; GFX11-TRUE16-NEXT: v_or_b16 v0.h, v0.h, v1.h
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v0
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v5.l
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v3 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v6 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_pk_sub_u16 v1, v2, v1 clamp
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: v_pk_lshrrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.h
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_and_or_b32 v0, 0xff, v0, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX11-TRUE16-NEXT: v_or3_b32 v0, v0, v1, v2
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_usubsat_v4i8:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v2, 8, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b16 v3, 8, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v0
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v2, v3, v2 clamp
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v5
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v6
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v1, v7, v1 clamp
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v0, v0, v4 clamp
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v2, 8, v2
-; GFX11-FAKE16-NEXT: v_sub_nc_u16 v3, v5, v3 clamp
-; GFX11-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 8, v1
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 24, v0
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 24, v1
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v2, v2, 16, v4
+; GFX11-FAKE16-NEXT: v_lshl_or_b32 v3, v3, 16, v5
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v0, v6, v0, 16
+; GFX11-FAKE16-NEXT: v_alignbit_b32 v1, v7, v1, 16
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v2, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v3, 8, v3 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
+; GFX11-FAKE16-NEXT: v_pk_sub_u16 v0, v0, v1 clamp
+; GFX11-FAKE16-NEXT: v_pk_lshrrev_b16 v1, 8, v2 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_pk_lshrrev_b16 v0, 8, v0 op_sel_hi:[0,1]
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 8
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v0, v0, 16, 8
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 24, v0
+; GFX11-FAKE16-NEXT: v_and_or_b32 v1, 0xff, v1, v2
+; GFX11-FAKE16-NEXT: v_or3_b32 v0, v1, v3, v0
; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
@@ -660,110 +725,299 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
define amdgpu_ps i32 @s_usubsat_v4i8(i32 inreg %lhs.arg, i32 inreg %rhs.arg) {
; GFX6-LABEL: s_usubsat_v4i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_and_b32 s2, s1, 0xff
-; GFX6-NEXT: s_and_b32 s3, s0, 0xff
-; GFX6-NEXT: s_lshr_b32 s4, s1, 24
-; GFX6-NEXT: s_lshr_b32 s5, s0, 24
-; GFX6-NEXT: s_bfe_u32 s6, s1, 0x80008
-; GFX6-NEXT: s_bfe_u32 s7, s0, 0x80008
-; GFX6-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX6-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX6-NEXT: s_max_u32 s0, s0, s1
+; GFX6-NEXT: s_lshr_b32 s2, s0, 8
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_lshr_b32 s4, s0, 24
+; GFX6-NEXT: s_lshr_b32 s5, s1, 8
+; GFX6-NEXT: s_lshr_b32 s6, s1, 16
+; GFX6-NEXT: s_lshr_b32 s7, s1, 24
+; GFX6-NEXT: s_lshl_b32 s0, s0, 24
+; GFX6-NEXT: s_lshl_b32 s1, s1, 24
+; GFX6-NEXT: s_min_u32 s1, s0, s1
; GFX6-NEXT: s_sub_i32 s0, s0, s1
-; GFX6-NEXT: s_max_u32 s1, s3, s2
-; GFX6-NEXT: s_max_u32 s5, s5, s4
+; GFX6-NEXT: s_lshl_b32 s1, s2, 24
+; GFX6-NEXT: s_lshl_b32 s2, s5, 24
+; GFX6-NEXT: s_min_u32 s2, s1, s2
; GFX6-NEXT: s_sub_i32 s1, s1, s2
-; GFX6-NEXT: s_max_u32 s2, s7, s6
-; GFX6-NEXT: s_sub_i32 s4, s5, s4
-; GFX6-NEXT: s_sub_i32 s2, s2, s6
-; GFX6-NEXT: s_lshl_b32 s4, s4, 24
-; GFX6-NEXT: s_lshl_b32 s0, s0, 16
-; GFX6-NEXT: s_lshl_b32 s2, s2, 8
-; GFX6-NEXT: s_or_b32 s0, s4, s0
-; GFX6-NEXT: s_or_b32 s1, s1, s2
-; GFX6-NEXT: s_or_b32 s0, s1, s0
+; GFX6-NEXT: s_lshl_b32 s2, s3, 24
+; GFX6-NEXT: s_lshl_b32 s3, s6, 24
+; GFX6-NEXT: s_min_u32 s3, s2, s3
+; GFX6-NEXT: s_sub_i32 s2, s2, s3
+; GFX6-NEXT: s_lshl_b32 s3, s4, 24
+; GFX6-NEXT: s_lshl_b32 s4, s7, 24
+; GFX6-NEXT: s_lshr_b32 s1, s1, 24
+; GFX6-NEXT: s_min_u32 s4, s3, s4
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: s_lshr_b32 s2, s2, 24
+; GFX6-NEXT: s_sub_i32 s3, s3, s4
+; GFX6-NEXT: v_alignbit_b32 v0, s1, v0, 24
+; GFX6-NEXT: s_lshr_b32 s3, s3, 24
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s1
+; GFX6-NEXT: s_lshl_b32 s1, s3, 24
+; GFX6-NEXT: s_or_b32 s0, s0, s1
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_usubsat_v4i8:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 24
-; GFX8-NEXT: s_lshr_b32 s3, s0, 24
-; GFX8-NEXT: s_max_u32 s3, s3, s2
-; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX8-NEXT: s_bfe_u32 s5, s0, 0x80008
-; GFX8-NEXT: s_sub_i32 s2, s3, s2
-; GFX8-NEXT: s_bfe_u32 s3, s1, 0x80010
-; GFX8-NEXT: s_bfe_u32 s6, s0, 0x80010
-; GFX8-NEXT: s_and_b32 s1, s1, 0xff
-; GFX8-NEXT: s_and_b32 s0, s0, 0xff
-; GFX8-NEXT: s_max_u32 s0, s0, s1
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_max_u32 s1, s5, s4
-; GFX8-NEXT: s_max_u32 s6, s6, s3
-; GFX8-NEXT: s_sub_i32 s1, s1, s4
-; GFX8-NEXT: s_lshl_b32 s2, s2, 8
-; GFX8-NEXT: s_sub_i32 s3, s6, s3
+; GFX8-NEXT: s_lshr_b32 s5, s1, 8
+; GFX8-NEXT: s_lshr_b32 s6, s1, 16
+; GFX8-NEXT: s_lshr_b32 s7, s1, 24
+; GFX8-NEXT: s_lshl_b32 s1, s1, 8
+; GFX8-NEXT: s_lshr_b32 s2, s0, 8
+; GFX8-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-NEXT: s_lshr_b32 s4, s0, 24
+; GFX8-NEXT: s_lshl_b32 s0, s0, 8
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s1, s2, 8
+; GFX8-NEXT: s_lshl_b32 s2, s5, 8
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_sub_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s2, s3, 8
+; GFX8-NEXT: s_lshl_b32 s3, s6, 8
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_sub_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: s_lshl_b32 s3, s4, 8
+; GFX8-NEXT: s_lshl_b32 s4, s7, 8
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: v_sub_u16_e64 v0, s3, v0 clamp
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshr_b32 s1, s1, 8
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_lshr_b32 s0, s0, 8
+; GFX8-NEXT: s_lshr_b32 s2, s2, 8
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
; GFX8-NEXT: s_lshl_b32 s1, s1, 8
-; GFX8-NEXT: s_or_b32 s2, s3, s2
+; GFX8-NEXT: s_lshr_b32 s3, s3, 8
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
+; GFX8-NEXT: s_lshl_b32 s1, s3, 24
; GFX8-NEXT: s_or_b32 s0, s0, s1
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX8-NEXT: s_or_b32 s0, s0, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_v4i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_lshr_b32 s2, s1, 24
-; GFX9-NEXT: s_lshr_b32 s3, s0, 24
-; GFX9-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX9-NEXT: s_bfe_u32 s5, s0, 0x80008
-; GFX9-NEXT: s_and_b32 s6, s1, 0xff
-; GFX9-NEXT: s_and_b32 s7, s0, 0xff
-; GFX9-NEXT: s_max_u32 s5, s5, s4
-; GFX9-NEXT: s_max_u32 s3, s3, s2
-; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX9-NEXT: s_max_u32 s7, s7, s6
-; GFX9-NEXT: s_sub_i32 s4, s5, s4
-; GFX9-NEXT: s_sub_i32 s2, s3, s2
-; GFX9-NEXT: s_max_u32 s0, s0, s1
-; GFX9-NEXT: s_sub_i32 s6, s7, s6
+; GFX9-NEXT: s_lshr_b32 s2, s0, 8
+; GFX9-NEXT: s_lshr_b32 s3, s0, 16
+; GFX9-NEXT: s_lshr_b32 s4, s0, 24
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX9-NEXT: s_lshr_b32 s4, s0, 16
+; GFX9-NEXT: s_lshl_b32 s0, s0, 0x80008
; GFX9-NEXT: s_lshl_b32 s4, s4, 8
-; GFX9-NEXT: s_lshl_b32 s2, s2, 8
-; GFX9-NEXT: s_sub_i32 s0, s0, s1
-; GFX9-NEXT: s_or_b32 s4, s6, s4
+; GFX9-NEXT: s_lshr_b32 s5, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX9-NEXT: s_lshr_b32 s4, s2, 16
+; GFX9-NEXT: s_lshr_b32 s6, s1, 16
+; GFX9-NEXT: s_lshr_b32 s7, s1, 24
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX9-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX9-NEXT: s_lshr_b32 s4, s1, 16
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s6, s7
+; GFX9-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s1, s4
+; GFX9-NEXT: s_lshr_b32 s4, s3, 16
+; GFX9-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX9-NEXT: s_lshl_b32 s4, s4, 8
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: s_pack_ll_b32_b16 s3, s3, s4
+; GFX9-NEXT: v_pk_sub_u16 v0, s0, v0 clamp
+; GFX9-NEXT: v_readfirstlane_b32 s0, v0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_pk_sub_u16 v0, s2, v0 clamp
+; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX9-NEXT: s_lshr_b32 s0, s0, 16
+; GFX9-NEXT: v_readfirstlane_b32 s1, v0
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshr_b32 s0, s0, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xffff
+; GFX9-NEXT: s_lshr_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX9-NEXT: s_lshr_b32 s1, s1, 8
+; GFX9-NEXT: s_pack_ll_b32_b16 s1, s2, s1
+; GFX9-NEXT: s_and_b32 s2, s0, 0xff
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_lshl_b32 s0, s0, 8
+; GFX9-NEXT: s_or_b32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s2, s1, 0xff
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
; GFX9-NEXT: s_or_b32 s0, s0, s2
-; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
-; GFX9-NEXT: s_lshl_b32 s0, s0, 16
-; GFX9-NEXT: s_or_b32 s0, s4, s0
+; GFX9-NEXT: s_lshl_b32 s1, s1, 24
+; GFX9-NEXT: s_or_b32 s0, s0, s1
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX10PLUS-LABEL: s_usubsat_v4i8:
-; GFX10PLUS: ; %bb.0:
-; GFX10PLUS-NEXT: s_lshr_b32 s2, s1, 24
-; GFX10PLUS-NEXT: s_lshr_b32 s3, s0, 24
-; GFX10PLUS-NEXT: s_bfe_u32 s4, s1, 0x80008
-; GFX10PLUS-NEXT: s_bfe_u32 s5, s0, 0x80008
-; GFX10PLUS-NEXT: s_and_b32 s6, s1, 0xff
-; GFX10PLUS-NEXT: s_and_b32 s7, s0, 0xff
-; GFX10PLUS-NEXT: s_max_u32 s5, s5, s4
-; GFX10PLUS-NEXT: s_max_u32 s3, s3, s2
-; GFX10PLUS-NEXT: s_bfe_u32 s1, s1, 0x80010
-; GFX10PLUS-NEXT: s_bfe_u32 s0, s0, 0x80010
-; GFX10PLUS-NEXT: s_max_u32 s7, s7, s6
-; GFX10PLUS-NEXT: s_sub_i32 s4, s5, s4
-; GFX10PLUS-NEXT: s_sub_i32 s2, s3, s2
-; GFX10PLUS-NEXT: s_max_u32 s0, s0, s1
-; GFX10PLUS-NEXT: s_sub_i32 s5, s7, s6
-; GFX10PLUS-NEXT: s_lshl_b32 s4, s4, 8
-; GFX10PLUS-NEXT: s_lshl_b32 s2, s2, 8
-; GFX10PLUS-NEXT: s_sub_i32 s0, s0, s1
-; GFX10PLUS-NEXT: s_or_b32 s1, s5, s4
-; GFX10PLUS-NEXT: s_or_b32 s0, s0, s2
-; GFX10PLUS-NEXT: s_and_b32 s1, s1, 0xffff
-; GFX10PLUS-NEXT: s_lshl_b32 s0, s0, 16
-; GFX10PLUS-NEXT: s_or_b32 s0, s1, s0
-; GFX10PLUS-NEXT: ; return to shader part epilog
+; GFX10-LABEL: s_usubsat_v4i8:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_lshr_b32 s2, s0, 8
+; GFX10-NEXT: s_lshr_b32 s5, s1, 8
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s4, s0, 24
+; GFX10-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10-NEXT: s_lshr_b32 s7, s1, 24
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s2
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_pack_ll_b32_b16 s2, s3, s4
+; GFX10-NEXT: s_lshr_b32 s3, s0, 16
+; GFX10-NEXT: s_lshr_b32 s5, s1, 16
+; GFX10-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s4, s6, s7
+; GFX10-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s0, s3
+; GFX10-NEXT: s_lshr_b32 s3, s2, 16
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX10-NEXT: s_lshr_b32 s5, s4, 16
+; GFX10-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX10-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-NEXT: s_lshl_b32 s4, s4, 0x80008
+; GFX10-NEXT: s_lshl_b32 s5, s5, 8
+; GFX10-NEXT: v_pk_sub_u16 v0, s0, s1 clamp
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s3
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s4, s5
+; GFX10-NEXT: v_pk_sub_u16 v1, s0, s1 clamp
+; GFX10-NEXT: v_readfirstlane_b32 s0, v0
+; GFX10-NEXT: v_readfirstlane_b32 s1, v1
+; GFX10-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX10-NEXT: s_lshr_b32 s0, s0, 16
+; GFX10-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX10-NEXT: s_lshr_b32 s0, s0, 8
+; GFX10-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX10-NEXT: s_lshr_b32 s1, s1, 16
+; GFX10-NEXT: s_lshr_b32 s3, s3, 0x80008
+; GFX10-NEXT: s_lshr_b32 s1, s1, 8
+; GFX10-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX10-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX10-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX10-NEXT: s_and_b32 s0, s0, 0xff
+; GFX10-NEXT: s_lshl_b32 s2, s2, 8
+; GFX10-NEXT: s_and_b32 s3, s1, 0xff
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s2, s3, 16
+; GFX10-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX10-NEXT: s_or_b32 s0, s0, s2
+; GFX10-NEXT: s_lshl_b32 s1, s1, 24
+; GFX10-NEXT: s_or_b32 s0, s0, s1
+; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX11-TRUE16-LABEL: s_usubsat_v4i8:
+; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-TRUE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-TRUE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-TRUE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-TRUE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-TRUE16-NEXT: v_pk_sub_u16 v0, s2, s3 clamp
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-TRUE16-NEXT: v_pk_sub_u16 v1, s0, s1 clamp
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX11-TRUE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s3, 0x80008
+; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-TRUE16-NEXT: s_lshr_b32 s2, s0, 16
+; GFX11-TRUE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0xff
+; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s3, 0xff
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 24
+; GFX11-TRUE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-TRUE16-NEXT: ; return to shader part epilog
+;
+; GFX11-FAKE16-LABEL: s_usubsat_v4i8:
+; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s0, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s0, 24
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s1, 8
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 24
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s0, s2
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s0, s0, s3
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s1, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s2, 16
+; GFX11-FAKE16-NEXT: s_pack_hl_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s3, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s2, s2, s4
+; GFX11-FAKE16-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s3, s3, s5
+; GFX11-FAKE16-NEXT: s_lshr_b32 s5, s1, 16
+; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s4, s4, 8
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 0x80008
+; GFX11-FAKE16-NEXT: s_lshl_b32 s5, s5, 8
+; GFX11-FAKE16-NEXT: v_pk_sub_u16 v0, s2, s3 clamp
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s0, s4
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s1, s5
+; GFX11-FAKE16-NEXT: v_pk_sub_u16 v1, s0, s1 clamp
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX11-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s2, s2, 0x80008
+; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xffff
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 16
+; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s3, 0x80008
+; GFX11-FAKE16-NEXT: s_lshr_b32 s1, s1, 8
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s0, s2, s0
+; GFX11-FAKE16-NEXT: s_pack_ll_b32_b16 s1, s3, s1
+; GFX11-FAKE16-NEXT: s_bfe_u32 s2, s0, 0x80010
+; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xff
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 8
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xff
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s3, 16
+; GFX11-FAKE16-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s2
+; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 24
+; GFX11-FAKE16-NEXT: s_or_b32 s0, s0, s1
+; GFX11-FAKE16-NEXT: ; return to shader part epilog
%lhs = bitcast i32 %lhs.arg to <4 x i8>
%rhs = bitcast i32 %rhs.arg to <4 x i8>
%result = call <4 x i8> @llvm.usub.sat.v4i8(<4 x i8> %lhs, <4 x i8> %rhs)
@@ -1854,9 +2108,9 @@ define <2 x i16> @v_usubsat_v2i16(<2 x i16> %lhs, <2 x i16> %rhs) {
; GFX8-LABEL: v_usubsat_v2i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u16_sdwa v2, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v0, v0, v1 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-NEXT: v_sub_u16_e64 v2, v0, v1 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubsat_v2i16:
@@ -1895,16 +2149,18 @@ define amdgpu_ps i32 @s_usubsat_v2i16(<2 x i16> inreg %lhs, <2 x i16> inreg %rhs
;
; GFX8-LABEL: s_usubsat_v2i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s2, s1, 0xffff
-; GFX8-NEXT: s_and_b32 s3, s0, 0xffff
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_max_u32 s0, s0, s1
-; GFX8-NEXT: s_max_u32 s3, s3, s2
-; GFX8-NEXT: s_sub_i32 s0, s0, s1
-; GFX8-NEXT: s_sub_i32 s2, s3, s2
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s2, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_sub_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_v2i16:
@@ -1944,10 +2200,10 @@ define amdgpu_ps float @usubsat_v2i16_sv(<2 x i16> inreg %lhs, <2 x i16> %rhs) {
; GFX8-LABEL: usubsat_v2i16_sv:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_sub_u16_e64 v1, s0, v0 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v0, v2, v0 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: usubsat_v2i16_sv:
@@ -1984,10 +2240,10 @@ define amdgpu_ps float @usubsat_v2i16_vs(<2 x i16> %lhs, <2 x i16> inreg %rhs) {
; GFX8-LABEL: usubsat_v2i16_vs:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_lshr_b32 s1, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_sub_u16_sdwa v1, v0, v1 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_sub_u16_e64 v0, v0, s0 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_sub_u16_e64 v1, v0, s0 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: usubsat_v2i16_vs:
@@ -2048,12 +2304,12 @@ define <2 x float> @v_usubsat_v4i16(<4 x i16> %lhs, <4 x i16> %rhs) {
; GFX8-LABEL: v_usubsat_v4i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u16_sdwa v4, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v0, v0, v2 clamp
-; GFX8-NEXT: v_sub_u16_sdwa v2, v1, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v1, v1, v3 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_sub_u16_e64 v4, v0, v2 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v2 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e64 v2, v1, v3 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubsat_v4i16:
@@ -2109,26 +2365,30 @@ define amdgpu_ps <2 x i32> @s_usubsat_v4i16(<4 x i16> inreg %lhs, <4 x i16> inre
;
; GFX8-LABEL: s_usubsat_v4i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX8-NEXT: s_and_b32 s5, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_max_u32 s1, s1, s3
-; GFX8-NEXT: s_max_u32 s5, s5, s4
-; GFX8-NEXT: s_sub_i32 s1, s1, s3
-; GFX8-NEXT: s_sub_i32 s4, s5, s4
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_or_b32 s1, s4, s1
-; GFX8-NEXT: s_and_b32 s3, s2, 0xffff
-; GFX8-NEXT: s_and_b32 s4, s0, 0xffff
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_max_u32 s0, s0, s2
-; GFX8-NEXT: s_max_u32 s4, s4, s3
-; GFX8-NEXT: s_sub_i32 s0, s0, s2
-; GFX8-NEXT: s_sub_i32 s3, s4, s3
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s3, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: s_lshr_b32 s6, s2, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s4, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_sub_u16_e64 v0, s4, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: s_lshr_b32 s7, s3, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s5, s1, 16
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: v_sub_u16_e64 v0, s5, v0 clamp
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, s2
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s3
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s2, s2, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s2
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_v4i16:
@@ -2209,15 +2469,15 @@ define <3 x float> @v_usubsat_v6i16(<6 x i16> %lhs, <6 x i16> %rhs) {
; GFX8-LABEL: v_usubsat_v6i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u16_sdwa v6, v0, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v0, v0, v3 clamp
-; GFX8-NEXT: v_sub_u16_sdwa v3, v1, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v1, v1, v4 clamp
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX8-NEXT: v_sub_u16_sdwa v3, v2, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v2, v2, v5 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v6
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX8-NEXT: v_sub_u16_e64 v6, v0, v3 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v3 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e64 v3, v1, v4 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e64 v4, v2, v5 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v2, v2, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v4, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubsat_v6i16:
@@ -2289,36 +2549,42 @@ define amdgpu_ps <3 x i32> @s_usubsat_v6i16(<6 x i16> inreg %lhs, <6 x i16> inre
;
; GFX8-LABEL: s_usubsat_v6i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s6, s5, 0xffff
-; GFX8-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX8-NEXT: s_lshr_b32 s5, s5, 16
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_max_u32 s2, s2, s5
-; GFX8-NEXT: s_max_u32 s7, s7, s6
-; GFX8-NEXT: s_sub_i32 s2, s2, s5
-; GFX8-NEXT: s_sub_i32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_or_b32 s2, s6, s2
-; GFX8-NEXT: s_and_b32 s5, s4, 0xffff
-; GFX8-NEXT: s_and_b32 s6, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_max_u32 s1, s1, s4
-; GFX8-NEXT: s_max_u32 s6, s6, s5
-; GFX8-NEXT: s_sub_i32 s1, s1, s4
-; GFX8-NEXT: s_sub_i32 s5, s6, s5
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_or_b32 s1, s5, s1
-; GFX8-NEXT: s_and_b32 s4, s3, 0xffff
-; GFX8-NEXT: s_and_b32 s5, s0, 0xffff
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_max_u32 s0, s0, s3
-; GFX8-NEXT: s_max_u32 s5, s5, s4
-; GFX8-NEXT: s_sub_i32 s0, s0, s3
-; GFX8-NEXT: s_sub_i32 s4, s5, s4
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s4, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: s_lshr_b32 s9, s3, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s6, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s9
+; GFX8-NEXT: v_sub_u16_e64 v0, s6, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_lshr_b32 s10, s4, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s7, s1, 16
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_sub_u16_e64 v0, s7, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
+; GFX8-NEXT: s_lshr_b32 s11, s5, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshr_b32 s8, s2, 16
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s11
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s8, v0 clamp
+; GFX8-NEXT: s_or_b32 s0, s0, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s4
+; GFX8-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s1, s1, s3
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s5
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s3
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_v6i16:
@@ -2405,18 +2671,18 @@ define <4 x float> @v_usubsat_v8i16(<8 x i16> %lhs, <8 x i16> %rhs) {
; GFX8-LABEL: v_usubsat_v8i16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_u16_sdwa v8, v0, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v0, v0, v4 clamp
-; GFX8-NEXT: v_sub_u16_sdwa v4, v1, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v1, v1, v5 clamp
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: v_sub_u16_sdwa v4, v2, v6 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v2, v2, v6 clamp
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX8-NEXT: v_sub_u16_sdwa v4, v3, v7 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_u16_e64 v3, v3, v7 clamp
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v8
-; GFX8-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX8-NEXT: v_sub_u16_e64 v8, v0, v4 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v0, v0, v4 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e64 v4, v1, v5 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v1, v1, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e64 v5, v2, v6 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v2, v2, v6 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_sub_u16_e64 v6, v3, v7 clamp
+; GFX8-NEXT: v_sub_u16_sdwa v3, v3, v7 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-NEXT: v_or_b32_e32 v0, v8, v0
+; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX8-NEXT: v_or_b32_e32 v3, v6, v3
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_usubsat_v8i16:
@@ -2504,46 +2770,54 @@ define amdgpu_ps <4 x i32> @s_usubsat_v8i16(<8 x i16> inreg %lhs, <8 x i16> inre
;
; GFX8-LABEL: s_usubsat_v8i16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_and_b32 s8, s7, 0xffff
-; GFX8-NEXT: s_and_b32 s9, s3, 0xffff
-; GFX8-NEXT: s_lshr_b32 s7, s7, 16
-; GFX8-NEXT: s_lshr_b32 s3, s3, 16
-; GFX8-NEXT: s_max_u32 s3, s3, s7
-; GFX8-NEXT: s_max_u32 s9, s9, s8
-; GFX8-NEXT: s_sub_i32 s3, s3, s7
-; GFX8-NEXT: s_sub_i32 s8, s9, s8
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_or_b32 s3, s8, s3
-; GFX8-NEXT: s_and_b32 s7, s6, 0xffff
-; GFX8-NEXT: s_and_b32 s8, s2, 0xffff
-; GFX8-NEXT: s_lshr_b32 s6, s6, 16
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_max_u32 s2, s2, s6
-; GFX8-NEXT: s_max_u32 s8, s8, s7
-; GFX8-NEXT: s_sub_i32 s2, s2, s6
-; GFX8-NEXT: s_sub_i32 s7, s8, s7
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: s_or_b32 s2, s7, s2
-; GFX8-NEXT: s_and_b32 s6, s5, 0xffff
-; GFX8-NEXT: s_and_b32 s7, s1, 0xffff
-; GFX8-NEXT: s_lshr_b32 s5, s5, 16
-; GFX8-NEXT: s_lshr_b32 s1, s1, 16
-; GFX8-NEXT: s_max_u32 s1, s1, s5
-; GFX8-NEXT: s_max_u32 s7, s7, s6
-; GFX8-NEXT: s_sub_i32 s1, s1, s5
-; GFX8-NEXT: s_sub_i32 s6, s7, s6
-; GFX8-NEXT: s_lshl_b32 s1, s1, 16
-; GFX8-NEXT: s_or_b32 s1, s6, s1
-; GFX8-NEXT: s_and_b32 s5, s4, 0xffff
-; GFX8-NEXT: s_and_b32 s6, s0, 0xffff
-; GFX8-NEXT: s_lshr_b32 s4, s4, 16
-; GFX8-NEXT: s_lshr_b32 s0, s0, 16
-; GFX8-NEXT: s_max_u32 s0, s0, s4
-; GFX8-NEXT: s_max_u32 s6, s6, s5
-; GFX8-NEXT: s_sub_i32 s0, s0, s4
-; GFX8-NEXT: s_sub_i32 s5, s6, s5
-; GFX8-NEXT: s_lshl_b32 s0, s0, 16
-; GFX8-NEXT: s_or_b32 s0, s5, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_lshr_b32 s12, s4, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s0, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s8, s0, 16
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s12
+; GFX8-NEXT: v_sub_u16_e64 v0, s8, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s5
+; GFX8-NEXT: s_lshr_b32 s13, s5, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s1, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s9, s1, 16
+; GFX8-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s13
+; GFX8-NEXT: v_sub_u16_e64 v0, s9, v0 clamp
+; GFX8-NEXT: v_readfirstlane_b32 s5, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: s_lshr_b32 s14, s6, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s2, v0 clamp
+; GFX8-NEXT: s_lshr_b32 s10, s2, 16
+; GFX8-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s14
+; GFX8-NEXT: v_sub_u16_e64 v0, s10, v0 clamp
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-NEXT: v_readfirstlane_b32 s6, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s7
+; GFX8-NEXT: s_and_b32 s0, 0xffff, s0
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_lshr_b32 s15, s7, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s3, v0 clamp
+; GFX8-NEXT: s_or_b32 s0, s0, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s5
+; GFX8-NEXT: s_lshr_b32 s11, s3, 16
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: v_mov_b32_e32 v0, s15
+; GFX8-NEXT: s_and_b32 s1, 0xffff, s1
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: v_sub_u16_e64 v0, s11, v0 clamp
+; GFX8-NEXT: s_or_b32 s1, s1, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s6
+; GFX8-NEXT: v_readfirstlane_b32 s7, v0
+; GFX8-NEXT: s_and_b32 s2, 0xffff, s2
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s2, s2, s4
+; GFX8-NEXT: s_and_b32 s4, 0xffff, s7
+; GFX8-NEXT: s_and_b32 s3, 0xffff, s3
+; GFX8-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-NEXT: s_or_b32 s3, s3, s4
; GFX8-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: s_usubsat_v8i16:
@@ -2586,8 +2860,14 @@ define i48 @v_usubsat_i48(i48 %lhs, i48 %rhs) {
; GFX6-NEXT: v_and_b32_e32 v3, 0xffff, v3
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v1, v2
; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, 0, vcc
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_usubsat_i48:
@@ -2632,14 +2912,21 @@ define i48 @v_usubsat_i48(i48 %lhs, i48 %rhs) {
define amdgpu_ps i48 @s_usubsat_i48(i48 inreg %lhs, i48 inreg %rhs) {
; GFX6-LABEL: s_usubsat_i48:
; GFX6: ; %bb.0:
+; GFX6-NEXT: s_sub_u32 s0, s0, s2
+; GFX6-NEXT: s_cselect_b32 s2, 1, 0
; GFX6-NEXT: s_and_b32 s1, s1, 0xffff
; GFX6-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX6-NEXT: s_sub_u32 s2, s0, s2
-; GFX6-NEXT: s_subb_u32 s3, s1, s3
-; GFX6-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX6-NEXT: s_cselect_b32 s1, 0, s3
-; GFX6-NEXT: s_cselect_b32 s0, 0, s2
+; GFX6-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-NEXT: s_subb_u32 s2, s1, s3
+; GFX6-NEXT: s_and_b32 s1, s2, 0xffff
+; GFX6-NEXT: s_cmp_lg_u32 s2, s1
+; GFX6-NEXT: s_cselect_b32 s2, 1, 0
+; GFX6-NEXT: s_lshr_b32 s3, s0, 16
+; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX6-NEXT: s_lshl_b32 s3, s3, 16
+; GFX6-NEXT: s_or_b32 s0, s0, s3
+; GFX6-NEXT: s_cmp_lg_u32 s2, 0
+; GFX6-NEXT: s_cselect_b64 s[0:1], 0, s[0:1]
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: s_usubsat_i48:
@@ -2683,8 +2970,14 @@ define amdgpu_ps <2 x float> @usubsat_i48_sv(i48 inreg %lhs, i48 %rhs) {
; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v1, vcc
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v1, v2
; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, 0, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_i48_sv:
@@ -2735,8 +3028,14 @@ define amdgpu_ps <2 x float> @usubsat_i48_vs(i48 %lhs, i48 inreg %rhs) {
; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-NEXT: v_and_b32_e32 v2, 0xffff, v1
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, v1, v2
; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, 0, vcc
-; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
+; GFX6-NEXT: v_cndmask_b32_e64 v1, v2, 0, vcc
; GFX6-NEXT: ; return to shader part epilog
;
; GFX8-LABEL: usubsat_i48_vs:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
index a9121d36b9d33..c819ef7a7f6dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.06 < %s | FileCheck --check-prefix=GFX906 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.06 < %s | FileCheck --check-prefix=GFX906 %s
define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1) %src2, ptr addrspace(1) nocapture %dst) {
; GFX906-LABEL: v3i8_liveout:
@@ -8,29 +8,38 @@ define amdgpu_kernel void @v3i8_liveout(ptr addrspace(1) %src1, ptr addrspace(1)
; GFX906-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX906-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX906-NEXT: v_mov_b32_e32 v3, 8
-; GFX906-NEXT: s_mov_b32 s4, 0xff0000
+; GFX906-NEXT: v_mov_b32_e32 v5, 16
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: global_load_dword v4, v2, s[0:1]
-; GFX906-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-NEXT: v_mov_b32_e32 v1, 0xff
; GFX906-NEXT: v_cmp_gt_u32_e32 vcc, 15, v0
; GFX906-NEXT: s_waitcnt vmcnt(0)
-; GFX906-NEXT: v_lshrrev_b32_sdwa v5, v3, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX906-NEXT: v_or_b32_sdwa v5, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX906-NEXT: v_and_or_b32 v4, v4, s4, v5
+; GFX906-NEXT: v_and_b32_e32 v6, 0xff, v4
+; GFX906-NEXT: v_lshlrev_b32_sdwa v7, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX906-NEXT: v_lshlrev_b32_sdwa v4, v5, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX906-NEXT: v_or3_b32 v4, v6, v7, v4
; GFX906-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX906-NEXT: s_cbranch_execz .LBB0_2
; GFX906-NEXT: ; %bb.1: ; %bb.1
; GFX906-NEXT: global_load_dword v0, v2, s[2:3]
; GFX906-NEXT: s_waitcnt vmcnt(0)
-; GFX906-NEXT: v_lshrrev_b32_sdwa v2, v3, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX906-NEXT: v_or_b32_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX906-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX906-NEXT: v_and_or_b32 v4, v0, s4, v2
+; GFX906-NEXT: v_and_b32_e32 v2, 0xff, v0
+; GFX906-NEXT: v_lshlrev_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX906-NEXT: v_lshlrev_b32_sdwa v0, v5, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_2
+; GFX906-NEXT: v_or3_b32 v4, v2, v3, v0
; GFX906-NEXT: .LBB0_2: ; %bb.2
; GFX906-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX906-NEXT: global_store_byte_d16_hi v1, v4, s[6:7] offset:2
-; GFX906-NEXT: global_store_short v1, v4, s[6:7]
+; GFX906-NEXT: v_lshrrev_b32_e32 v0, 8, v4
+; GFX906-NEXT: v_and_b32_e32 v0, 0xff, v0
+; GFX906-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GFX906-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX906-NEXT: v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX906-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX906-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX906-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GFX906-NEXT: v_mov_b32_e32 v1, 0
+; GFX906-NEXT: global_store_short v1, v0, s[6:7]
+; GFX906-NEXT: global_store_byte_d16_hi v1, v0, s[6:7] offset:2
; GFX906-NEXT: s_endpgm
entry:
%idx = call i32 @llvm.amdgcn.workitem.id.x()
@@ -577,23 +586,38 @@ define amdgpu_kernel void @v32i8_loop_carried(ptr addrspace(1) %src1, ptr addrsp
; GFX906: ; %bb.0: ; %entry
; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX906-NEXT: v_lshlrev_b32_e32 v1, 5, v0
-; GFX906-NEXT: v_cmp_lt_u32_e32 vcc, 14, v0
-; GFX906-NEXT: s_mov_b32 s2, 0x2000604
+; GFX906-NEXT: v_mov_b32_e32 v2, 0xff
+; GFX906-NEXT: v_cmp_le_u32_e32 vcc, 15, v0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: global_load_dword v1, v1, s[0:1]
; GFX906-NEXT: s_mov_b64 s[0:1], 0
; GFX906-NEXT: s_waitcnt vmcnt(0)
-; GFX906-NEXT: v_mov_b32_e32 v0, v1
+; GFX906-NEXT: v_and_b32_sdwa v0, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX906-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX906-NEXT: v_and_or_b32 v3, v1, v2, v0
+; GFX906-NEXT: v_mov_b32_e32 v4, v1
; GFX906-NEXT: .LBB10_1: ; %bb.1
; GFX906-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX906-NEXT: s_and_b64 s[6:7], exec, vcc
-; GFX906-NEXT: s_or_b64 s[0:1], s[6:7], s[0:1]
-; GFX906-NEXT: v_perm_b32 v0, v1, v0, s2
+; GFX906-NEXT: v_mov_b32_e32 v0, v4
+; GFX906-NEXT: v_and_b32_e32 v4, 0xff, v0
+; GFX906-NEXT: s_and_b64 s[2:3], exec, vcc
+; GFX906-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX906-NEXT: v_and_b32_sdwa v5, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX906-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GFX906-NEXT: v_or3_b32 v4, v3, v4, v5
; GFX906-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX906-NEXT: s_cbranch_execnz .LBB10_1
; GFX906-NEXT: ; %bb.2: ; %bb.2.loopexit
; GFX906-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX906-NEXT: v_mov_b32_e32 v2, 0xff
; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX906-NEXT: v_and_b32_sdwa v3, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX906-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX906-NEXT: v_and_or_b32 v1, v1, v2, v3
+; GFX906-NEXT: v_and_b32_e32 v3, 0xff, v0
+; GFX906-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX906-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX906-NEXT: v_or3_b32 v0, v1, v3, v0
; GFX906-NEXT: v_mov_b32_e32 v1, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: global_store_dword v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
index e60f62c26ab5e..b8b323aa83851 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-contents-legalization.ll
@@ -2187,10 +2187,15 @@ define void @store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v4i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -2248,10 +2253,15 @@ define void @store_v5i8(<5 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v5i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_mov_b32_e32 v6, 8
+; GISEL-NEXT: v_mov_b32_e32 v5, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v5, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: buffer_store_byte v4, off, s[16:19], 0 offset:4
; GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -2397,12 +2407,19 @@ define void @store_v7i8(<7 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v7i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_mov_b32_e32 v8, 8
+; GISEL-NEXT: v_mov_b32_e32 v7, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v8, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v7, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
-; GISEL-NEXT: v_perm_b32 v0, v4, v5, s4
+; GISEL-NEXT: v_and_b32_e32 v0, 0xff, v5
+; GISEL-NEXT: v_lshlrev_b16_e32 v0, 8, v0
+; GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GISEL-NEXT: buffer_store_short v0, off, s[16:19], 0 offset:4
; GISEL-NEXT: buffer_store_byte v6, off, s[16:19], 0 offset:6
; GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -2472,14 +2489,23 @@ define void @store_v8i8(<8 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v8i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v4, v4, v5, s4
-; GISEL-NEXT: v_perm_b32 v5, v6, v7, s4
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v4, v5, 16, v4
-; GISEL-NEXT: v_lshl_or_b32 v3, v1, 16, v0
-; GISEL-NEXT: buffer_store_dwordx2 v[3:4], off, s[16:19], 0
+; GISEL-NEXT: v_mov_b32_e32 v9, 8
+; GISEL-NEXT: v_mov_b32_e32 v8, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v8, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v9, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v7
+; GISEL-NEXT: v_and_or_b32 v1, v4, v8, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GISEL-NEXT: v_or3_b32 v1, v1, v2, v3
+; GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2563,17 +2589,30 @@ define void @store_v12i8(<12 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v12i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v8, v8, v9, s4
-; GISEL-NEXT: v_perm_b32 v9, v10, v11, s4
-; GISEL-NEXT: v_perm_b32 v4, v4, v5, s4
-; GISEL-NEXT: v_perm_b32 v5, v6, v7, s4
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v8, v9, 16, v8
-; GISEL-NEXT: v_lshl_or_b32 v7, v5, 16, v4
-; GISEL-NEXT: v_lshl_or_b32 v6, v1, 16, v0
-; GISEL-NEXT: buffer_store_dwordx3 v[6:8], off, s[16:19], 0
+; GISEL-NEXT: v_mov_b32_e32 v13, 8
+; GISEL-NEXT: v_mov_b32_e32 v12, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v13, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v12, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v13, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v7
+; GISEL-NEXT: v_and_or_b32 v1, v4, v12, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GISEL-NEXT: v_or3_b32 v1, v1, v2, v3
+; GISEL-NEXT: v_lshlrev_b32_sdwa v2, v13, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v10
+; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v11
+; GISEL-NEXT: v_and_or_b32 v2, v8, v12, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; GISEL-NEXT: v_or3_b32 v2, v2, v3, v4
+; GISEL-NEXT: buffer_store_dwordx3 v[0:2], off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2673,20 +2712,37 @@ define void @store_v16i8(<16 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v16i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v12, v12, v13, s4
-; GISEL-NEXT: v_perm_b32 v13, v14, v15, s4
-; GISEL-NEXT: v_perm_b32 v8, v8, v9, s4
-; GISEL-NEXT: v_perm_b32 v9, v10, v11, s4
-; GISEL-NEXT: v_perm_b32 v4, v4, v5, s4
-; GISEL-NEXT: v_perm_b32 v5, v6, v7, s4
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v12, v13, 16, v12
-; GISEL-NEXT: v_lshl_or_b32 v11, v9, 16, v8
-; GISEL-NEXT: v_lshl_or_b32 v10, v5, 16, v4
-; GISEL-NEXT: v_lshl_or_b32 v9, v1, 16, v0
-; GISEL-NEXT: buffer_store_dwordx4 v[9:12], off, s[16:19], 0
+; GISEL-NEXT: v_mov_b32_e32 v17, 8
+; GISEL-NEXT: v_mov_b32_e32 v16, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v17, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v16, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v17, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v6
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v7
+; GISEL-NEXT: v_and_or_b32 v1, v4, v16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GISEL-NEXT: v_or3_b32 v1, v1, v2, v3
+; GISEL-NEXT: v_lshlrev_b32_sdwa v2, v17, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v10
+; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v11
+; GISEL-NEXT: v_and_or_b32 v2, v8, v16, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; GISEL-NEXT: v_or3_b32 v2, v2, v3, v4
+; GISEL-NEXT: v_lshlrev_b32_sdwa v3, v17, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v14
+; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v15
+; GISEL-NEXT: v_and_or_b32 v3, v12, v16, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; GISEL-NEXT: v_or3_b32 v3, v3, v4, v5
+; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -2850,35 +2906,67 @@ define void @store_v32i8(<32 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: store_v32i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v8, v8, v9, s4
-; GISEL-NEXT: v_perm_b32 v9, v10, v11, s4
-; GISEL-NEXT: buffer_load_ubyte v10, off, s[0:3], s32
-; GISEL-NEXT: v_perm_b32 v12, v12, v13, s4
-; GISEL-NEXT: v_perm_b32 v13, v14, v15, s4
-; GISEL-NEXT: v_perm_b32 v4, v4, v5, s4
-; GISEL-NEXT: v_perm_b32 v5, v6, v7, s4
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v6, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v3, v13, 16, v12
-; GISEL-NEXT: v_lshl_or_b32 v2, v9, 16, v8
-; GISEL-NEXT: v_lshl_or_b32 v1, v5, 16, v4
-; GISEL-NEXT: v_lshl_or_b32 v0, v6, 16, v0
-; GISEL-NEXT: v_perm_b32 v7, v28, v29, s4
-; GISEL-NEXT: v_perm_b32 v11, v24, v25, s4
-; GISEL-NEXT: v_perm_b32 v14, v26, v27, s4
-; GISEL-NEXT: v_perm_b32 v15, v20, v21, s4
-; GISEL-NEXT: v_perm_b32 v20, v22, v23, s4
-; GISEL-NEXT: v_perm_b32 v16, v16, v17, s4
-; GISEL-NEXT: v_perm_b32 v17, v18, v19, s4
+; GISEL-NEXT: v_mov_b32_e32 v31, 8
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v31, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_mov_b32_e32 v32, 0xff
+; GISEL-NEXT: v_and_or_b32 v0, v0, v32, v1
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v31, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v7
+; GISEL-NEXT: buffer_load_ubyte v7, off, s[0:3], s32
+; GISEL-NEXT: v_and_or_b32 v1, v4, v32, v1
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v3
+; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v6
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; GISEL-NEXT: v_or3_b32 v0, v0, v2, v3
+; GISEL-NEXT: v_or3_b32 v1, v1, v4, v5
+; GISEL-NEXT: v_lshlrev_b32_sdwa v2, v31, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v3, 0xff, v10
+; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v11
+; GISEL-NEXT: v_and_or_b32 v2, v8, v32, v2
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; GISEL-NEXT: v_or3_b32 v2, v2, v3, v4
+; GISEL-NEXT: v_lshlrev_b32_sdwa v3, v31, v13 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v4, 0xff, v14
+; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v15
+; GISEL-NEXT: v_and_or_b32 v3, v12, v32, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 24, v5
+; GISEL-NEXT: v_or3_b32 v3, v3, v4, v5
+; GISEL-NEXT: v_lshlrev_b32_sdwa v4, v31, v17 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v18
+; GISEL-NEXT: v_and_b32_e32 v6, 0xff, v19
+; GISEL-NEXT: v_and_or_b32 v4, v16, v32, v4
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GISEL-NEXT: v_lshlrev_b32_sdwa v8, v31, v21 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_or3_b32 v4, v4, v5, v6
+; GISEL-NEXT: v_and_b32_e32 v5, 0xff, v22
+; GISEL-NEXT: v_and_b32_e32 v6, 0xff, v23
+; GISEL-NEXT: v_and_or_b32 v8, v20, v32, v8
+; GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GISEL-NEXT: v_lshlrev_b32_e32 v6, 24, v6
+; GISEL-NEXT: v_or3_b32 v5, v8, v5, v6
+; GISEL-NEXT: v_lshlrev_b32_sdwa v6, v31, v25 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v8, 0xff, v26
+; GISEL-NEXT: v_and_b32_e32 v9, 0xff, v27
+; GISEL-NEXT: v_and_or_b32 v6, v24, v32, v6
+; GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; GISEL-NEXT: v_lshlrev_b32_e32 v9, 24, v9
+; GISEL-NEXT: v_or3_b32 v6, v6, v8, v9
+; GISEL-NEXT: v_lshlrev_b32_sdwa v8, v31, v29 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_b32_e32 v9, 0xff, v30
+; GISEL-NEXT: v_and_or_b32 v8, v28, v32, v8
+; GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GISEL-NEXT: s_waitcnt vmcnt(0)
+; GISEL-NEXT: v_lshlrev_b32_e32 v7, 24, v7
+; GISEL-NEXT: v_or3_b32 v7, v8, v9, v7
; GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[16:19], 0
-; GISEL-NEXT: v_lshl_or_b32 v5, v14, 16, v11
-; GISEL-NEXT: v_lshl_or_b32 v4, v20, 16, v15
-; GISEL-NEXT: v_lshl_or_b32 v3, v17, 16, v16
-; GISEL-NEXT: s_waitcnt vmcnt(1)
-; GISEL-NEXT: v_perm_b32 v0, v30, v10, s4
-; GISEL-NEXT: v_lshl_or_b32 v6, v0, 16, v7
-; GISEL-NEXT: buffer_store_dwordx4 v[3:6], off, s[16:19], 0 offset:16
+; GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[16:19], 0 offset:16
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3068,6 +3156,23 @@ define i40 @load_i40(ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-NEXT: buffer_load_dword v0, off, s[16:19], 0
; GISEL-NEXT: buffer_load_ubyte v1, off, s[16:19], 0 offset:4
+; GISEL-NEXT: s_waitcnt vmcnt(1)
+; GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; GISEL-NEXT: s_lshr_b32 s5, s4, 8
+; GISEL-NEXT: s_lshr_b32 s6, s4, 16
+; GISEL-NEXT: s_lshr_b32 s7, s4, 24
+; GISEL-NEXT: s_and_b32 s5, s5, 0xff
+; GISEL-NEXT: s_and_b32 s6, s6, 0xff
+; GISEL-NEXT: s_lshl_b32 s7, s7, 8
+; GISEL-NEXT: s_and_b32 s4, s4, 0xff
+; GISEL-NEXT: s_lshl_b32 s5, s5, 8
+; GISEL-NEXT: s_or_b32 s6, s6, s7
+; GISEL-NEXT: s_or_b32 s4, s4, s5
+; GISEL-NEXT: s_and_b32 s5, 0xffff, s6
+; GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GISEL-NEXT: s_lshl_b32 s5, s5, 16
+; GISEL-NEXT: s_or_b32 s4, s4, s5
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
%p = addrspacecast ptr addrspace(8) %buf to ptr addrspace(7)
@@ -3508,19 +3613,20 @@ define void @store_v8i4(<8 x i4> %data, ptr addrspace(8) inreg %buf) {
; GISEL-NEXT: v_and_b32_e32 v1, 15, v1
; GISEL-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GISEL-NEXT: v_and_or_b32 v0, v0, 15, v1
-; GISEL-NEXT: v_and_b32_e32 v1, 15, v3
-; GISEL-NEXT: v_and_b32_e32 v2, 15, v2
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 12, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
-; GISEL-NEXT: v_and_b32_e32 v1, 15, v5
-; GISEL-NEXT: v_mov_b32_e32 v2, 15
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 20, v1
-; GISEL-NEXT: v_and_b32_sdwa v3, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GISEL-NEXT: v_or3_b32 v0, v0, v3, v1
-; GISEL-NEXT: v_lshlrev_b32_e32 v1, 28, v7
-; GISEL-NEXT: v_and_b32_sdwa v2, v6, v2 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GISEL-NEXT: v_or3_b32 v0, v0, v2, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 15, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 15, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 12, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GISEL-NEXT: v_mov_b32_e32 v1, 15
+; GISEL-NEXT: v_and_b32_e32 v3, 15, v5
+; GISEL-NEXT: v_and_b32_sdwa v2, v4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GISEL-NEXT: v_lshlrev_b32_e32 v3, 20, v3
+; GISEL-NEXT: v_or3_b32 v0, v0, v2, v3
+; GISEL-NEXT: v_and_b32_e32 v2, 15, v7
+; GISEL-NEXT: v_and_b32_sdwa v1, v6, v1 dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 28, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -3676,10 +3782,15 @@ define void @volatile_store_v4i8(<4 x i8> %data, ptr addrspace(8) inreg %buf) {
; GISEL-LABEL: volatile_store_v4i8:
; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b32 s4, 0xc0c0004
-; GISEL-NEXT: v_perm_b32 v0, v0, v1, s4
-; GISEL-NEXT: v_perm_b32 v1, v2, v3, s4
-; GISEL-NEXT: v_lshl_or_b32 v0, v1, 16, v0
+; GISEL-NEXT: v_mov_b32_e32 v5, 8
+; GISEL-NEXT: v_mov_b32_e32 v4, 0xff
+; GISEL-NEXT: v_lshlrev_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GISEL-NEXT: v_and_or_b32 v0, v0, v4, v1
+; GISEL-NEXT: v_and_b32_e32 v1, 0xff, v2
+; GISEL-NEXT: v_and_b32_e32 v2, 0xff, v3
+; GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
; GISEL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GISEL-NEXT: s_waitcnt vmcnt(0)
; GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
index 0c414f6b87646..39f482ff71a22 100644
--- a/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
+++ b/llvm/test/CodeGen/AMDGPU/build-vector-packed-partial-undef.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16,GFX11-FAKE16-SDAG %s
@@ -9,14 +9,24 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-TRUE16,GFX11-TRUE16-GISEL %s
define void @undef_lo_v2i16(i16 %arg0) {
-; GFX8-LABEL: undef_lo_v2i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo_v2i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo_v2i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_lo_v2i16:
; GFX9-SDAG: ; %bb.0:
@@ -72,14 +82,24 @@ define void @undef_lo_v2i16(i16 %arg0) {
}
define void @undef_lo_v2f16(half %arg0) {
-; GFX8-LABEL: undef_lo_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo_v2f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo_v2f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_lo_v2f16:
; GFX9-SDAG: ; %bb.0:
@@ -135,16 +155,31 @@ define void @undef_lo_v2f16(half %arg0) {
}
define void @undef_lo_op_v2f16(half %arg0) {
-; GFX8-LABEL: undef_lo_op_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x3c00
-; GFX8-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, 0x7e00, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo_op_v2f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, 0x3c00
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, 0x7e00, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo_op_v2f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e64 v1, s4, 1.0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX8-GISEL-NEXT: v_add_f16_e32 v0, 1.0, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_lo_op_v2f16:
; GFX9-SDAG: ; %bb.0:
@@ -208,15 +243,27 @@ define void @undef_lo_op_v2f16(half %arg0) {
}
define void @undef_lo_op_v2i16(i16 %arg0) {
-; GFX8-LABEL: undef_lo_op_v2i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x63
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo_op_v2i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, 0x63
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo_op_v2i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x63
+; GFX8-GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_lo_op_v2i16:
; GFX9-SDAG: ; %bb.0:
@@ -293,14 +340,25 @@ define void @undef_lo_op_v2i16(i16 %arg0) {
}
define void @undef_lo3_v4i16(i16 %arg0) {
-; GFX8-LABEL: undef_lo3_v4i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v[0:1]
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo3_v4i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v[0:1]
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo3_v4i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v[0:1]
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_lo3_v4i16:
; GFX9-SDAG: ; %bb.0:
@@ -356,14 +414,25 @@ define void @undef_lo3_v4i16(i16 %arg0) {
}
define void @undef_lo3_v4f16(half %arg0) {
-; GFX8-LABEL: undef_lo3_v4f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v[0:1]
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_lo3_v4f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v[0:1]
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_lo3_v4f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v[0:1]
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_lo3_v4f16:
; GFX9-SDAG: ; %bb.0:
@@ -661,15 +730,30 @@ define void @undef_hi_v2f16(half %arg0) {
}
define void @undef_hi_op_v2f16(half %arg0) {
-; GFX8-LABEL: undef_hi_op_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_f16_e32 v0, 1.0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, 0x7e000000, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_hi_op_v2f16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_f16_e32 v0, 1.0, v0
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, 0x7e000000, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_hi_op_v2f16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e64 v1, s4, 1.0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; GFX8-GISEL-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-GISEL-NEXT: v_add_f16_e32 v0, 1.0, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s4
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_hi_op_v2f16:
; GFX9-SDAG: ; %bb.0:
@@ -737,14 +821,26 @@ define void @undef_hi_op_v2f16(half %arg0) {
}
define void @undef_hi_op_v2i16(i16 %arg0) {
-; GFX8-LABEL: undef_hi_op_v2i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_u16_e32 v0, 0x63, v0
-; GFX8-NEXT: ;;#ASMSTART
-; GFX8-NEXT: ; use v0
-; GFX8-NEXT: ;;#ASMEND
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: undef_hi_op_v2i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_u16_e32 v0, 0x63, v0
+; GFX8-SDAG-NEXT: ;;#ASMSTART
+; GFX8-SDAG-NEXT: ; use v0
+; GFX8-SDAG-NEXT: ;;#ASMEND
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: undef_hi_op_v2i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX8-GISEL-NEXT: v_add_u16_e32 v0, 0x63, v0
+; GFX8-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, s4, v0
+; GFX8-GISEL-NEXT: ;;#ASMSTART
+; GFX8-GISEL-NEXT: ; use v0
+; GFX8-GISEL-NEXT: ;;#ASMEND
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: undef_hi_op_v2i16:
; GFX9-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index 7cb4adfabed3c..e987b04b6e3bc 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgpu6.00 < %s | FileCheck -enable-var-scope -check-prefixes=SI %s
; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=VI %s
; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -enable-var-scope -check-prefix=EG %s
-; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope -check-prefixes=SI-GISEL %s
-; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope -check-prefixes=VI-GISEL %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -enable-var-scope -check-prefixes=SI-GISEL %s
+; RUN: llc -mtriple=amdgpu8.02 -mattr=-flat-for-global -global-isel=1 < %s | FileCheck -enable-var-scope -check-prefixes=VI-GISEL %s
declare i16 @llvm.ctpop.i16(i16) nounwind readnone
declare <2 x i16> @llvm.ctpop.v2i16(<2 x i16>) nounwind readnone
@@ -538,44 +538,43 @@ define amdgpu_kernel void @v_ctpop_v2i16(ptr addrspace(1) noalias %out, ptr addr
; SI-GISEL-LABEL: v_ctpop_v2i16:
; SI-GISEL: ; %bb.0:
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s10, 0
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b64 s[8:9], s[2:3]
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; SI-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
-; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-GISEL-LABEL: v_ctpop_v2i16:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_dword v0, v[0:1]
-; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
; VI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; VI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-GISEL-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
@@ -703,57 +702,56 @@ define amdgpu_kernel void @v_ctpop_v4i16(ptr addrspace(1) noalias %out, ptr addr
; SI-GISEL-LABEL: v_ctpop_v4i16:
; SI-GISEL: ; %bb.0:
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s10, 0
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b64 s[8:9], s[2:3]
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; SI-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[8:11], 0 addr64
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v3, v3, 0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-GISEL-NEXT: v_or_b32_e32 v1, v3, v1
-; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
-; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v3, v3, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-GISEL-LABEL: v_ctpop_v4i16:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 3, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; VI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; VI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
; VI-GISEL-NEXT: v_bcnt_u32_b32 v2, v2, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v3, v3, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; VI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-GISEL-NEXT: v_or_b32_e32 v1, v1, v3
; VI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -939,82 +937,81 @@ define amdgpu_kernel void @v_ctpop_v8i16(ptr addrspace(1) noalias %out, ptr addr
;
; SI-GISEL-LABEL: v_ctpop_v8i16:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s10, 0
-; SI-GISEL-NEXT: s_mov_b32 s11, s3
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b64 s[8:9], s[6:7]
; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; SI-GISEL-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[8:11], 0 addr64
-; SI-GISEL-NEXT: s_mov_b32 s2, -1
-; SI-GISEL-NEXT: s_mov_b32 s0, s4
-; SI-GISEL-NEXT: s_mov_b32 s1, s5
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dwordx4 v[0:3], v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b32 s6, -1
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v0
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v1
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v2
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v3
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v3, v3, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v7, v7, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v6, v6, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v5, v5, 0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v4, v4, 0
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-GISEL-NEXT: v_or_b32_e32 v3, v7, v3
-; SI-GISEL-NEXT: v_or_b32_e32 v2, v6, v2
-; SI-GISEL-NEXT: v_or_b32_e32 v1, v5, v1
-; SI-GISEL-NEXT: v_or_b32_e32 v0, v4, v0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v5, v5, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v6, v6, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v7, v7, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v3, v3, 0
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; SI-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
; SI-GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-GISEL-LABEL: v_ctpop_v8i16:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 4, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 4, v0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, s2, v0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; VI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; VI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; VI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
; VI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; VI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; VI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
; VI-GISEL-NEXT: v_bcnt_u32_b32 v4, v4, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v5, v5, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v6, v6, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v7, v7, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v3, v3, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v2, v2, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v2, v2, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v3, v3, 0
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; VI-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
-; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v5
-; VI-GISEL-NEXT: v_or_b32_e32 v1, v1, v6
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v7
+; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; VI-GISEL-NEXT: v_or_b32_e32 v1, v1, v5
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; VI-GISEL-NEXT: v_or_b32_e32 v3, v3, v7
; VI-GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
@@ -1335,113 +1332,112 @@ define amdgpu_kernel void @v_ctpop_v16i16(ptr addrspace(1) noalias %out, ptr add
;
; SI-GISEL-LABEL: v_ctpop_v16i16:
; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s10, 0
-; SI-GISEL-NEXT: s_mov_b32 s11, s3
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-GISEL-NEXT: v_lshlrev_b32_e32 v4, 5, v0
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b64 s[8:9], s[6:7]
; SI-GISEL-NEXT: v_mov_b32_e32 v5, 0
-; SI-GISEL-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[8:11], 0 addr64 offset:16
-; SI-GISEL-NEXT: buffer_load_dwordx4 v[4:7], v[4:5], s[8:11], 0 addr64
-; SI-GISEL-NEXT: s_mov_b32 s2, -1
-; SI-GISEL-NEXT: s_mov_b32 s0, s4
-; SI-GISEL-NEXT: s_mov_b32 s1, s5
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64
+; SI-GISEL-NEXT: buffer_load_dwordx4 v[4:7], v[4:5], s[4:7], 0 addr64 offset:16
+; SI-GISEL-NEXT: s_mov_b32 s6, -1
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
; SI-GISEL-NEXT: s_waitcnt vmcnt(1)
-; SI-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v0
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-GISEL-NEXT: v_and_b32_e32 v9, 0xffff, v1
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; SI-GISEL-NEXT: v_and_b32_e32 v10, 0xffff, v2
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; SI-GISEL-NEXT: v_and_b32_e32 v11, 0xffff, v3
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_and_b32_e32 v12, 0xffff, v4
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; SI-GISEL-NEXT: v_and_b32_e32 v13, 0xffff, v5
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v5
-; SI-GISEL-NEXT: v_and_b32_e32 v14, 0xffff, v6
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; SI-GISEL-NEXT: v_and_b32_e32 v15, 0xffff, v7
-; SI-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v7, v7, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v6, v6, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v5, v5, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v4, v4, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v3, v3, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; SI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v8, v8, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v9, v9, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v10, v10, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v11, v11, 0
+; SI-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; SI-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; SI-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; SI-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v0, v0, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v15, v15, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v14, v14, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v13, v13, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v1, v1, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v2, v2, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v3, v3, 0
; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v12, v12, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v11, v11, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v10, v10, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v9, v9, 0
-; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v8, v8, 0
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v16, 16, v3
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v17, 16, v2
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v18, 16, v1
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v19, 16, v0
-; SI-GISEL-NEXT: v_or_b32_e32 v3, v15, v7
-; SI-GISEL-NEXT: v_or_b32_e32 v2, v14, v6
-; SI-GISEL-NEXT: v_or_b32_e32 v1, v13, v5
-; SI-GISEL-NEXT: v_or_b32_e32 v0, v12, v4
-; SI-GISEL-NEXT: v_or_b32_e32 v7, v11, v16
-; SI-GISEL-NEXT: v_or_b32_e32 v6, v10, v17
-; SI-GISEL-NEXT: v_or_b32_e32 v5, v9, v18
-; SI-GISEL-NEXT: v_or_b32_e32 v4, v8, v19
-; SI-GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v13, v13, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v14, v14, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v15, v15, 0
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v8
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v4, v4, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v5, v5, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v6, v6, 0
+; SI-GISEL-NEXT: v_bcnt_u32_b32_e64 v7, v7, 0
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v12, 16, v12
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v13, 16, v13
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v14, 16, v14
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v15, 16, v15
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; SI-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; SI-GISEL-NEXT: v_or_b32_e32 v4, v4, v12
+; SI-GISEL-NEXT: v_or_b32_e32 v5, v5, v13
+; SI-GISEL-NEXT: v_or_b32_e32 v6, v6, v14
+; SI-GISEL-NEXT: v_or_b32_e32 v7, v7, v15
; SI-GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
; SI-GISEL-NEXT: s_endpgm
;
; VI-GISEL-LABEL: v_ctpop_v16i16:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 5, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 5, v0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; VI-GISEL-NEXT: v_add_u32_e32 v4, vcc, s2, v0
+; VI-GISEL-NEXT: v_add_u32_e32 v4, vcc, v0, v2
; VI-GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v1, vcc
; VI-GISEL-NEXT: flat_load_dwordx4 v[0:3], v[4:5]
; VI-GISEL-NEXT: v_add_u32_e32 v4, vcc, 16, v4
; VI-GISEL-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc
; VI-GISEL-NEXT: flat_load_dwordx4 v[4:7], v[4:5]
-; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
; VI-GISEL-NEXT: s_waitcnt vmcnt(1)
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v0
-; VI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; VI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; VI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v8, 16, v0
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v2
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; VI-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; VI-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; VI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; VI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v7
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v6
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; VI-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v14, 16, v6
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; VI-GISEL-NEXT: v_bcnt_u32_b32 v8, v8, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v9, v9, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v10, v10, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v11, v11, 0
-; VI-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
-; VI-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; VI-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; VI-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v3, v3, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v2, v2, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v0, v0, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v1, v1, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v2, v2, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v3, v3, 0
+; VI-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; VI-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; VI-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; VI-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v7
; VI-GISEL-NEXT: v_bcnt_u32_b32 v12, v12, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v13, v13, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v14, v14, 0
@@ -1450,22 +1446,22 @@ define amdgpu_kernel void @v_ctpop_v16i16(ptr addrspace(1) noalias %out, ptr add
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v9
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
; VI-GISEL-NEXT: v_lshlrev_b32_e32 v11, 16, v11
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v7, v7, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v6, v6, 0
-; VI-GISEL-NEXT: v_bcnt_u32_b32 v5, v5, 0
; VI-GISEL-NEXT: v_bcnt_u32_b32 v4, v4, 0
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v12, 16, v12
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v13, 16, v13
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v14, 16, v14
-; VI-GISEL-NEXT: v_lshlrev_b32_e32 v15, 16, v15
-; VI-GISEL-NEXT: v_or_b32_e32 v3, v3, v8
-; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v9
-; VI-GISEL-NEXT: v_or_b32_e32 v1, v1, v10
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v11
-; VI-GISEL-NEXT: v_or_b32_e32 v7, v7, v12
-; VI-GISEL-NEXT: v_or_b32_e32 v6, v6, v13
-; VI-GISEL-NEXT: v_or_b32_e32 v5, v5, v14
-; VI-GISEL-NEXT: v_or_b32_e32 v4, v4, v15
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v5, v5, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v6, v6, 0
+; VI-GISEL-NEXT: v_bcnt_u32_b32 v7, v7, 0
+; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; VI-GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; VI-GISEL-NEXT: v_or_b32_e32 v3, v3, v11
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v8, 16, v12
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v9, 16, v13
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v14
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v11, 16, v15
+; VI-GISEL-NEXT: v_or_b32_e32 v4, v4, v8
+; VI-GISEL-NEXT: v_or_b32_e32 v5, v5, v9
+; VI-GISEL-NEXT: v_or_b32_e32 v6, v6, v10
+; VI-GISEL-NEXT: v_or_b32_e32 v7, v7, v11
; VI-GISEL-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; VI-GISEL-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
; VI-GISEL-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
index cc72f3e342fdc..fdfdb56db0f82 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fmul-sel.ll
@@ -1688,27 +1688,47 @@ define half @fmul_select_f16_test2(half %x, i32 %bool.arg1, i32 %bool.arg2) {
}
define <2 x half> @fmul_select_v2f16_test3(<2 x half> %x, <2 x i32> %bool.arg1, <2 x i32> %bool.arg2) {
-; GFX7-LABEL: fmul_select_v2f16_test3:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v6, 0x3c00
-; GFX7-NEXT: v_mov_b32_e32 v7, 0x4000
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v5
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_v2f16_test3:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v6, 0x3c00
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v7, 0x4000
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v5
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: fmul_select_v2f16_test3:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
+; GFX7-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_ldexp_f32_e32 v1, v3, v2
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fmul_select_v2f16_test3:
; GFX9: ; %bb.0:
@@ -1795,27 +1815,47 @@ define <2 x half> @fmul_select_v2f16_test3(<2 x half> %x, <2 x i32> %bool.arg1,
}
define <2 x half> @fmul_select_v2f16_test4(<2 x half> %x, <2 x i32> %bool.arg1, <2 x i32> %bool.arg2) {
-; GFX7-LABEL: fmul_select_v2f16_test4:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v6, 0x3c00
-; GFX7-NEXT: v_mov_b32_e32 v7, 0x3800
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
-; GFX7-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
-; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX7-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v5
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_mul_f32_e32 v2, v3, v2
-; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX7-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: fmul_select_v2f16_test4:
+; GFX7-SDAG: ; %bb.0:
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v6, 0x3c00
+; GFX7-SDAG-NEXT: v_mov_b32_e32 v7, 0x3800
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v1, v6, v7, vcc
+; GFX7-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX7-SDAG-NEXT: v_cndmask_b32_e32 v2, v6, v7, vcc
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v5
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v2, v3, v2
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX7-SDAG-NEXT: v_mul_f32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GISEL-LABEL: fmul_select_v2f16_test4:
+; GFX7-GISEL: ; %bb.0:
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v3
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-GISEL-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
+; GFX7-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v4
+; GFX7-GISEL-NEXT: v_cndmask_b32_e64 v2, 0, -1, vcc
+; GFX7-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_ldexp_f32_e32 v1, v3, v2
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX7-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: fmul_select_v2f16_test4:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index 845fd53895074..ca1b3ed8c32dd 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GFX678,GFX678-SDAG,GFX6-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GFX678,GFX678-GISEL,GFX6-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu7.01-amd-amdhsa < %s | FileCheck -check-prefixes=GFX678,GFX678-GISEL,GFX6-GISEL %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=GFX678,GFX678-SDAG,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=GFX678,GFX678-GISEL,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu8.03-amd-amdhsa < %s | FileCheck -check-prefixes=GFX678,GFX678-GISEL,GFX8-GISEL %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -amdgpu-scalarize-global-loads=false -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -amdgpu-scalarize-global-loads=false -mtriple=amdgpu11.00-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-TRUE16 %s
@@ -6453,22 +6453,21 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_flush(ptr addrspace(1)
; GFX6-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX6-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-GISEL-NEXT: flat_load_dword v0, v[0:1]
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-GISEL-NEXT: v_or_b32_e32 v4, v0, v1
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX6-GISEL-NEXT: flat_store_dword v[0:1], v4
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-GISEL-NEXT: v_bfe_u32 v3, v3, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-GISEL-NEXT: flat_store_dword v[0:1], v3
; GFX6-GISEL-NEXT: s_endpgm
;
; GFX8-SDAG-LABEL: test_canonicalize_value_v2f16_flush:
@@ -6502,19 +6501,23 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_flush(ptr addrspace(1)
; GFX8-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX8-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 0x3c00
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, 16
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GISEL-NEXT: flat_load_dword v0, v[0:1]
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x3c00
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX8-GISEL-NEXT: v_mul_f16_sdwa v1, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-GISEL-NEXT: v_mul_f16_e32 v0, 1.0, v0
-; GFX8-GISEL-NEXT: v_or_b32_e32 v4, v0, v1
-; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX8-GISEL-NEXT: flat_store_dword v[0:1], v4
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v6, 1.0, v3
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: flat_store_dword v[0:1], v3
; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: test_canonicalize_value_v2f16_flush:
@@ -7112,22 +7115,21 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(ptr addrspace(1)
; GFX6-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX6-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
+; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX6-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v2
; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-GISEL-NEXT: flat_load_dword v0, v[0:1]
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-GISEL-NEXT: v_or_b32_e32 v4, v0, v1
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX6-GISEL-NEXT: flat_store_dword v[0:1], v4
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v3
+; GFX6-GISEL-NEXT: v_bfe_u32 v3, v3, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX6-GISEL-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-GISEL-NEXT: flat_store_dword v[0:1], v3
; GFX6-GISEL-NEXT: s_endpgm
;
; GFX8-SDAG-LABEL: test_canonicalize_value_v2f16_denorm:
@@ -7160,17 +7162,21 @@ define amdgpu_kernel void @test_canonicalize_value_v2f16_denorm(ptr addrspace(1)
; GFX8-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX8-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX8-GISEL-NEXT: flat_load_dword v0, v[0:1]
+; GFX8-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX8-GISEL-NEXT: v_max_f16_sdwa v3, v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v0
-; GFX8-GISEL-NEXT: v_or_b32_e32 v3, v0, v3
-; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; GFX8-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
; GFX8-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e32 v5, v3, v3
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v3, v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-GISEL-NEXT: flat_store_dword v[0:1], v3
; GFX8-GISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 5e3e2ca025bc7..25090bf693552 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG %s
@@ -2207,25 +2207,49 @@ define amdgpu_ps float @test_fmax_f32_s_ieee_off(float inreg %a, float inreg %b)
}
define amdgpu_kernel void @test_fmax_v2f16_v_ieee_on(ptr addrspace(1) %out, <2 x half> %a, <2 x half> %b) #0 {
-; GFX8-LABEL: test_fmax_v2f16_v_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s4, s0
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s0, s0
-; GFX8-NEXT: s_lshr_b32 s0, s2, 16
-; GFX8-NEXT: v_max_f16_e64 v1, s0, s0
-; GFX8-NEXT: v_max_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e64 v1, s3, s3
-; GFX8-NEXT: v_max_f16_e64 v2, s2, s2
-; GFX8-NEXT: v_max_f16_e32 v1, v2, v1
-; GFX8-NEXT: s_mov_b32 s5, s1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmax_v2f16_v_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s0, s0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-SDAG-NEXT: v_max_f16_e64 v2, s2, s2
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX8-SDAG-NEXT: s_endpgm
+;
+; GFX8-GISEL-LABEL: test_fmax_v2f16_v_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s2, s2
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-GISEL-NEXT: s_lshr_b32 s4, s2, 16
+; GFX8-GISEL-NEXT: s_lshr_b32 s5, s3, 16
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s5, s5
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s3, s3, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: test_fmax_v2f16_v_ieee_on:
; GFX9-SDAG: ; %bb.0:
@@ -2284,12 +2308,21 @@ define amdgpu_kernel void @test_fmax_v2f16_v_ieee_on(ptr addrspace(1) %out, <2 x
}
define amdgpu_ps <2 x half> @test_fmax_v2f16_v_ieee_off(<2 x half> %a, <2 x half> %b) #0 {
-; GFX8-LABEL: test_fmax_v2f16_v_ieee_off:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_max_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_max_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: test_fmax_v2f16_v_ieee_off:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: test_fmax_v2f16_v_ieee_off:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_max_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: test_fmax_v2f16_v_ieee_off:
; GFX9: ; %bb.0:
@@ -2305,25 +2338,49 @@ define amdgpu_ps <2 x half> @test_fmax_v2f16_v_ieee_off(<2 x half> %a, <2 x half
}
define amdgpu_kernel void @test_fmax_v2f16_s_ieee_on(ptr addrspace(1) %out, <2 x half> inreg %a, <2 x half> inreg %b) #0 {
-; GFX8-LABEL: test_fmax_v2f16_s_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s4, s0
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s0, s0
-; GFX8-NEXT: s_lshr_b32 s0, s2, 16
-; GFX8-NEXT: v_max_f16_e64 v1, s0, s0
-; GFX8-NEXT: v_max_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e64 v1, s3, s3
-; GFX8-NEXT: v_max_f16_e64 v2, s2, s2
-; GFX8-NEXT: v_max_f16_e32 v1, v2, v1
-; GFX8-NEXT: s_mov_b32 s5, s1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmax_v2f16_s_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s0, s0
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-SDAG-NEXT: v_max_f16_e64 v2, s2, s2
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX8-SDAG-NEXT: s_endpgm
+;
+; GFX8-GISEL-LABEL: test_fmax_v2f16_s_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s2, s2
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-GISEL-NEXT: s_lshr_b32 s4, s2, 16
+; GFX8-GISEL-NEXT: s_lshr_b32 s5, s3, 16
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s5, s5
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s3, s3, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: test_fmax_v2f16_s_ieee_on:
; GFX9-SDAG: ; %bb.0:
@@ -2382,17 +2439,34 @@ define amdgpu_kernel void @test_fmax_v2f16_s_ieee_on(ptr addrspace(1) %out, <2 x
}
define amdgpu_ps <2 x half> @test_fmax_v2f16_s_ieee_off(<2 x half> inreg %a, <2 x half> inreg %b) #0 {
-; GFX8-LABEL: test_fmax_v2f16_s_ieee_off:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_max_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_max_f16_e32 v1, s0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: test_fmax_v2f16_s_ieee_off:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_lshr_b32 s2, s1, 16
+; GFX8-SDAG-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-SDAG-NEXT: v_max_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT: v_max_f16_e32 v1, s0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: test_fmax_v2f16_s_ieee_off:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-GISEL-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-GISEL-NEXT: v_max_f16_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: test_fmax_v2f16_s_ieee_off:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.ll b/llvm/test/CodeGen/AMDGPU/fmed3.ll
index 9b0d10061d170..c4960317f1a7d 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -mtriple=amdgpu6.00 -global-isel=0 < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-SDAG %s
-; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-GISEL %s
+; RUN: llc -mtriple=amdgpu6.00 -global-isel=1 < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-GISEL %s
; RUN: llc -mtriple=amdgpu8.02 -global-isel=0 < %s | FileCheck -enable-var-scope -check-prefixes=VI-SDAG %s
-; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 -global-isel-abort=2 < %s | FileCheck -enable-var-scope -check-prefixes=VI-GISEL %s
+; RUN: llc -mtriple=amdgpu8.02 -global-isel=1 < %s | FileCheck -enable-var-scope -check-prefixes=VI-GISEL %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=0 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -mtriple=amdgpu9.00 -global-isel=1 < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -mtriple=amdgpu11.00 -global-isel=0 -mattr=-real-true16 < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
@@ -8784,25 +8784,47 @@ define half @v_test_fmed3_r_i_i_f16_minimumnum_maximumnum(half %a) {
}
define <2 x half> @v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum(<2 x half> %a) {
-; SI-LABEL: v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_max_f32_e32 v1, 2.0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_max_f32_e32 v0, 2.0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_min_f32_e32 v1, 4.0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_min_f32_e32 v0, 4.0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_max_f32_e32 v1, 2.0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_max_f32_e32 v0, 2.0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_min_f32_e32 v1, 4.0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_min_f32_e32 v0, 4.0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_max_f32_e32 v1, 2.0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_max_f32_e32 v0, 2.0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_min_f32_e32 v1, 4.0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_min_f32_e32 v0, 4.0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum:
; VI-SDAG: ; %bb.0:
@@ -8820,14 +8842,15 @@ define <2 x half> @v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum(<2 x half> %a)
; VI-GISEL-LABEL: v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-GISEL-NEXT: v_max_f16_sdwa v1, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; VI-GISEL-NEXT: v_max_f16_e32 v0, v0, v0
-; VI-GISEL-NEXT: v_max_f16_e32 v1, 2.0, v1
+; VI-GISEL-NEXT: v_max_f16_e32 v1, v0, v0
+; VI-GISEL-NEXT: v_max_f16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; VI-GISEL-NEXT: v_max_f16_e32 v0, 2.0, v0
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x4400
+; VI-GISEL-NEXT: v_max_f16_e32 v1, 2.0, v1
; VI-GISEL-NEXT: v_min_f16_e32 v0, 4.0, v0
-; VI-GISEL-NEXT: v_min_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_min_f16_e32 v1, 4.0, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_test_fmed3_r_i_i_v2f16_minimumnum_maximumnum:
@@ -8940,40 +8963,77 @@ define float @v_test_nnan_input_fmed3_r_i_i_f32_maximum_minimum(float %a) {
}
define <2 x half> @v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum(<2 x half> %a) {
-; SI-LABEL: v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v0
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mov_b32_e32 v2, 0x7fc00000
-; SI-NEXT: v_add_f32_e32 v1, 1.0, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_max_f32_e32 v3, 2.0, v1
-; SI-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
-; SI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; SI-NEXT: v_max_f32_e32 v3, 2.0, v0
-; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_min_f32_e32 v4, 4.0, v0
-; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
-; SI-NEXT: v_min_f32_e32 v3, 4.0, v1
-; SI-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
-; SI-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7fc00000
+; SI-SDAG-NEXT: v_add_f32_e32 v1, 1.0, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_add_f32_e32 v0, 1.0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_max_f32_e32 v3, 2.0, v1
+; SI-SDAG-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; SI-SDAG-NEXT: v_max_f32_e32 v3, 2.0, v0
+; SI-SDAG-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_min_f32_e32 v4, 4.0, v0
+; SI-SDAG-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
+; SI-SDAG-NEXT: v_min_f32_e32 v3, 4.0, v1
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc
+; SI-SDAG-NEXT: v_cmp_o_f32_e32 vcc, v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v2, 0x7fc00000
+; SI-GISEL-NEXT: v_add_f32_e32 v1, 1.0, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_add_f32_e32 v0, 1.0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_max_f32_e32 v3, 2.0, v1
+; SI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 2.0, v1
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; SI-GISEL-NEXT: v_max_f32_e32 v3, 2.0, v0
+; SI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 2.0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_min_f32_e32 v3, 4.0, v1
+; SI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 4.0, v1
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
+; SI-GISEL-NEXT: v_min_f32_e32 v3, 4.0, v0
+; SI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 4.0, v0
+; SI-GISEL-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; VI-SDAG-LABEL: v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum:
; VI-SDAG: ; %bb.0:
@@ -8992,15 +9052,33 @@ define <2 x half> @v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum(<2 x half
; VI-GISEL-LABEL: v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3c00
; VI-GISEL-NEXT: v_add_f16_e32 v1, 1.0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3c00
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
; VI-GISEL-NEXT: v_add_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-GISEL-NEXT: v_max_f16_e32 v0, 2.0, v0
-; VI-GISEL-NEXT: v_max_f16_e32 v1, 2.0, v1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x4400
-; VI-GISEL-NEXT: v_min_f16_e32 v1, 4.0, v1
-; VI-GISEL-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fc00000
+; VI-GISEL-NEXT: v_max_f32_e32 v2, 2.0, v1
+; VI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 2.0, v1
+; VI-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; VI-GISEL-NEXT: v_max_f32_e32 v2, 2.0, v0
+; VI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 2.0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_min_f32_e32 v2, 4.0, v1
+; VI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 4.0, v1
+; VI-GISEL-NEXT: v_cndmask_b32_e32 v1, v3, v2, vcc
+; VI-GISEL-NEXT: v_min_f32_e32 v2, 4.0, v0
+; VI-GISEL-NEXT: v_cmp_o_f32_e32 vcc, 4.0, v0
+; VI-GISEL-NEXT: v_cndmask_b32_e32 v0, v3, v2, vcc
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_test_nnan_input_fmed3_r_i_i_v2f16_maximum_minimum:
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index 7a58abb97d531..06fb492792ae8 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.02 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.08 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-flat-for-global < %s | FileCheck -enable-var-scope -check-prefixes=GFX12,GFX12-SDAG %s
@@ -2120,25 +2120,49 @@ define amdgpu_ps float @test_fmin_f32_s_ieee_off(float inreg %a, float inreg %b)
}
define amdgpu_kernel void @test_fmin_v2f16_v_ieee_on(ptr addrspace(1) %out, <2 x half> %a, <2 x half> %b) #0 {
-; GFX8-LABEL: test_fmin_v2f16_v_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s4, s0
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s0, s0
-; GFX8-NEXT: s_lshr_b32 s0, s2, 16
-; GFX8-NEXT: v_max_f16_e64 v1, s0, s0
-; GFX8-NEXT: v_min_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e64 v1, s3, s3
-; GFX8-NEXT: v_max_f16_e64 v2, s2, s2
-; GFX8-NEXT: v_min_f16_e32 v1, v2, v1
-; GFX8-NEXT: s_mov_b32 s5, s1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmin_v2f16_v_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s0, s0
+; GFX8-SDAG-NEXT: v_min_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-SDAG-NEXT: v_max_f16_e64 v2, s2, s2
+; GFX8-SDAG-NEXT: v_min_f16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX8-SDAG-NEXT: s_endpgm
+;
+; GFX8-GISEL-LABEL: test_fmin_v2f16_v_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s2, s2
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-GISEL-NEXT: s_lshr_b32 s4, s2, 16
+; GFX8-GISEL-NEXT: s_lshr_b32 s5, s3, 16
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s5, s5
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s3, s3, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: test_fmin_v2f16_v_ieee_on:
; GFX9-SDAG: ; %bb.0:
@@ -2197,12 +2221,21 @@ define amdgpu_kernel void @test_fmin_v2f16_v_ieee_on(ptr addrspace(1) %out, <2 x
}
define amdgpu_ps <2 x half> @test_fmin_v2f16_v_ieee_off(<2 x half> %a, <2 x half> %b) #0 {
-; GFX8-LABEL: test_fmin_v2f16_v_ieee_off:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: v_min_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_min_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: test_fmin_v2f16_v_ieee_off:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: v_min_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: test_fmin_v2f16_v_ieee_off:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_min_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_min_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: test_fmin_v2f16_v_ieee_off:
; GFX9: ; %bb.0:
@@ -2218,25 +2251,49 @@ define amdgpu_ps <2 x half> @test_fmin_v2f16_v_ieee_off(<2 x half> %a, <2 x half
}
define amdgpu_kernel void @test_fmin_v2f16_s_ieee_on(ptr addrspace(1) %out, <2 x half> inreg %a, <2 x half> inreg %b) #0 {
-; GFX8-LABEL: test_fmin_v2f16_s_ieee_on:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b32 s4, s0
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: v_max_f16_e64 v0, s0, s0
-; GFX8-NEXT: s_lshr_b32 s0, s2, 16
-; GFX8-NEXT: v_max_f16_e64 v1, s0, s0
-; GFX8-NEXT: v_min_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_max_f16_e64 v1, s3, s3
-; GFX8-NEXT: v_max_f16_e64 v2, s2, s2
-; GFX8-NEXT: v_min_f16_e32 v1, v2, v1
-; GFX8-NEXT: s_mov_b32 s5, s1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
-; GFX8-NEXT: s_endpgm
+; GFX8-SDAG-LABEL: test_fmin_v2f16_s_ieee_on:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s0, s0
+; GFX8-SDAG-NEXT: v_min_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-SDAG-NEXT: v_max_f16_e64 v2, s2, s2
+; GFX8-SDAG-NEXT: v_min_f16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX8-SDAG-NEXT: s_endpgm
+;
+; GFX8-GISEL-LABEL: test_fmin_v2f16_s_ieee_on:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s2, s2
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s3, s3
+; GFX8-GISEL-NEXT: s_lshr_b32 s4, s2, 16
+; GFX8-GISEL-NEXT: s_lshr_b32 s5, s3, 16
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_max_f16_e64 v1, s4, s4
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-GISEL-NEXT: v_max_f16_e64 v0, s5, s5
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s3, s3, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s2, s2, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; GFX8-GISEL-NEXT: s_or_b32 s2, s2, s3
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: test_fmin_v2f16_s_ieee_on:
; GFX9-SDAG: ; %bb.0:
@@ -2295,17 +2352,34 @@ define amdgpu_kernel void @test_fmin_v2f16_s_ieee_on(ptr addrspace(1) %out, <2 x
}
define amdgpu_ps <2 x half> @test_fmin_v2f16_s_ieee_off(<2 x half> inreg %a, <2 x half> inreg %b) #0 {
-; GFX8-LABEL: test_fmin_v2f16_s_ieee_off:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s2, s1, 16
-; GFX8-NEXT: s_lshr_b32 s3, s0, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_min_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_min_f16_e32 v1, s0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: test_fmin_v2f16_s_ieee_off:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_lshr_b32 s2, s1, 16
+; GFX8-SDAG-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-SDAG-NEXT: v_min_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT: v_min_f16_e32 v1, s0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: test_fmin_v2f16_s_ieee_off:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-GISEL-NEXT: s_lshr_b32 s3, s1, 16
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: s_lshr_b32 s2, s0, 16
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-GISEL-NEXT: v_min_f16_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX9-LABEL: test_fmin_v2f16_s_ieee_off:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
index 7b4dcaa1c3dab..9f639666f2ca5 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoi.i128.ll
@@ -3,93 +3,231 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00-amd-amdhsa < %s | FileCheck -check-prefixes=GCN,GISEL %s
define i128 @fptosi_f64_to_i128(double %x) {
-; GCN-LABEL: fptosi_f64_to_i128:
-; GCN: ; %bb.0: ; %fp-to-i-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v5, v1
-; GCN-NEXT: v_bfe_u32 v6, v5, 20, 11
-; GCN-NEXT: v_mov_b32_e32 v7, 0
-; GCN-NEXT: s_mov_b64 s[4:5], 0x3fe
-; GCN-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB0_6
-; GCN-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
-; GCN-NEXT: s_mov_b64 s[4:5], 0x432
-; GCN-NEXT: v_ashrrev_i32_e32 v8, 31, v5
-; GCN-NEXT: v_and_b32_e32 v0, 0xfffff, v5
-; GCN-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
-; GCN-NEXT: v_or_b32_e32 v9, 1, v8
-; GCN-NEXT: v_or_b32_e32 v5, 0x100000, v0
-; GCN-NEXT: ; implicit-def: $vgpr0_vgpr1
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB0_3
-; GCN-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
-; GCN-NEXT: v_add_u32_e32 v3, 0xfffffbcd, v6
-; GCN-NEXT: v_sub_u32_e32 v0, 0x473, v6
-; GCN-NEXT: v_add_u32_e32 v2, 0xfffffb8d, v6
-; GCN-NEXT: v_lshrrev_b64 v[0:1], v0, v[4:5]
-; GCN-NEXT: v_lshlrev_b64 v[6:7], v2, v[4:5]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
-; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v3
-; GCN-NEXT: v_lshlrev_b64 v[3:4], v3, v[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
-; GCN-NEXT: v_cndmask_b32_e32 v10, 0, v3, vcc
-; GCN-NEXT: v_cndmask_b32_e64 v7, 0, v1, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[4:5]
-; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v9, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_cndmask_b32_e32 v12, 0, v4, vcc
-; GCN-NEXT: v_mul_lo_u32 v11, v8, v5
-; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v9, v[1:2]
-; GCN-NEXT: v_mul_lo_u32 v7, v9, v7
-; GCN-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v5, 0
-; GCN-NEXT: v_mul_lo_u32 v9, v8, v10
-; GCN-NEXT: v_mov_b32_e32 v1, v3
-; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v8, v[1:2]
-; GCN-NEXT: v_add3_u32 v6, v6, v7, v11
-; GCN-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v8, v10, v[5:6]
-; GCN-NEXT: v_mul_lo_u32 v7, v8, v12
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GCN-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
-; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v8, v[2:3]
-; GCN-NEXT: v_add3_u32 v4, v9, v6, v7
-; GCN-NEXT: ; implicit-def: $vgpr6_vgpr7
-; GCN-NEXT: ; implicit-def: $vgpr9
-; GCN-NEXT: ; implicit-def: $vgpr8
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
-; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GCN-NEXT: ; implicit-def: $vgpr4_vgpr5
-; GCN-NEXT: .LBB0_3: ; %Flow
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB0_5
-; GCN-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
-; GCN-NEXT: v_sub_u32_e32 v0, 0x433, v6
-; GCN-NEXT: v_lshrrev_b64 v[4:5], v0, v[4:5]
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v4, v9, 0
-; GCN-NEXT: v_mad_u64_u32 v[6:7], s[8:9], v5, v9, v[1:2]
-; GCN-NEXT: v_mov_b32_e32 v1, v6
-; GCN-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v4, v8, v[1:2]
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v7, v2
-; GCN-NEXT: v_addc_co_u32_e64 v3, s[8:9], 0, 0, vcc
-; GCN-NEXT: v_mad_u64_u32 v[2:3], s[8:9], v5, v8, v[2:3]
-; GCN-NEXT: v_mul_lo_u32 v5, v8, v5
-; GCN-NEXT: v_mad_u64_u32 v[2:3], s[8:9], v8, v4, v[2:3]
-; GCN-NEXT: v_mul_lo_u32 v4, v8, v4
-; GCN-NEXT: v_add3_u32 v3, v4, v3, v5
-; GCN-NEXT: .LBB0_5: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB0_6: ; %fp-to-i-cleanup
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fptosi_f64_to_i128:
+; SDAG: ; %bb.0: ; %fp-to-i-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v5, v1
+; SDAG-NEXT: v_bfe_u32 v6, v5, 20, 11
+; SDAG-NEXT: v_mov_b32_e32 v7, 0
+; SDAG-NEXT: s_mov_b64 s[4:5], 0x3fe
+; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
+; SDAG-NEXT: v_mov_b32_e32 v4, v0
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB0_6
+; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; SDAG-NEXT: s_mov_b64 s[4:5], 0x432
+; SDAG-NEXT: v_ashrrev_i32_e32 v8, 31, v5
+; SDAG-NEXT: v_and_b32_e32 v0, 0xfffff, v5
+; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[4:5], v[6:7]
+; SDAG-NEXT: v_or_b32_e32 v9, 1, v8
+; SDAG-NEXT: v_or_b32_e32 v5, 0x100000, v0
+; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB0_3
+; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; SDAG-NEXT: v_add_u32_e32 v3, 0xfffffbcd, v6
+; SDAG-NEXT: v_sub_u32_e32 v0, 0x473, v6
+; SDAG-NEXT: v_add_u32_e32 v2, 0xfffffb8d, v6
+; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[4:5]
+; SDAG-NEXT: v_lshlrev_b64 v[6:7], v2, v[4:5]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
+; SDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v3
+; SDAG-NEXT: v_lshlrev_b64 v[3:4], v3, v[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
+; SDAG-NEXT: v_cndmask_b32_e32 v10, 0, v3, vcc
+; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v1, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[4:5]
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v9, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_cndmask_b32_e32 v12, 0, v4, vcc
+; SDAG-NEXT: v_mul_lo_u32 v11, v8, v5
+; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v12, v9, v[1:2]
+; SDAG-NEXT: v_mul_lo_u32 v7, v9, v7
+; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v5, 0
+; SDAG-NEXT: v_mul_lo_u32 v9, v8, v10
+; SDAG-NEXT: v_mov_b32_e32 v1, v3
+; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v10, v8, v[1:2]
+; SDAG-NEXT: v_add3_u32 v6, v6, v7, v11
+; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v8, v10, v[5:6]
+; SDAG-NEXT: v_mul_lo_u32 v7, v8, v12
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
+; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
+; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v8, v[2:3]
+; SDAG-NEXT: v_add3_u32 v4, v9, v6, v7
+; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
+; SDAG-NEXT: ; implicit-def: $vgpr9
+; SDAG-NEXT: ; implicit-def: $vgpr8
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
+; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
+; SDAG-NEXT: .LBB0_3: ; %Flow
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB0_5
+; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; SDAG-NEXT: v_sub_u32_e32 v0, 0x433, v6
+; SDAG-NEXT: v_lshrrev_b64 v[4:5], v0, v[4:5]
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v4, v9, 0
+; SDAG-NEXT: v_mad_u64_u32 v[6:7], s[8:9], v5, v9, v[1:2]
+; SDAG-NEXT: v_mov_b32_e32 v1, v6
+; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v4, v8, v[1:2]
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v7, v2
+; SDAG-NEXT: v_addc_co_u32_e64 v3, s[8:9], 0, 0, vcc
+; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[8:9], v5, v8, v[2:3]
+; SDAG-NEXT: v_mul_lo_u32 v5, v8, v5
+; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[8:9], v8, v4, v[2:3]
+; SDAG-NEXT: v_mul_lo_u32 v4, v8, v4
+; SDAG-NEXT: v_add3_u32 v3, v4, v3, v5
+; SDAG-NEXT: .LBB0_5: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB0_6: ; %fp-to-i-cleanup
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fptosi_f64_to_i128:
+; GISEL: ; %bb.0: ; %fp-to-i-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GISEL-NEXT: v_mov_b32_e32 v5, v1
+; GISEL-NEXT: v_lshrrev_b64 v[0:1], 52, v[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v7, 0
+; GISEL-NEXT: v_mov_b32_e32 v1, 0x3ff
+; GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-NEXT: v_bfe_u32 v6, v0, 0, 11
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[6:7], v[1:2]
+; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GISEL-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB0_6
+; GISEL-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; GISEL-NEXT: v_cmp_ge_i64_e32 vcc, -1, v[4:5]
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_lshlrev_b16_e32 v2, 1, v0
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GISEL-NEXT: v_lshlrev_b16_e32 v3, 2, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GISEL-NEXT: v_lshlrev_b16_e32 v8, 3, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v8
+; GISEL-NEXT: v_lshlrev_b16_e32 v9, 4, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v8
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GISEL-NEXT: v_lshlrev_b16_e32 v10, 5, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v9
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v10
+; GISEL-NEXT: v_lshlrev_b16_e32 v11, 6, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v11
+; GISEL-NEXT: v_lshlrev_b16_e32 v12, 7, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v11
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v12
+; GISEL-NEXT: v_lshlrev_b16_e32 v13, 8, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v12
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v13
+; GISEL-NEXT: v_lshlrev_b16_e32 v14, 9, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v13
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v14
+; GISEL-NEXT: v_lshlrev_b16_e32 v15, 10, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v14
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v15
+; GISEL-NEXT: v_lshlrev_b16_e32 v16, 11, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v15
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v16
+; GISEL-NEXT: v_lshlrev_b16_e32 v17, 12, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v16
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v17
+; GISEL-NEXT: v_lshlrev_b16_e32 v18, 13, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v17
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v18
+; GISEL-NEXT: v_lshlrev_b16_e32 v19, 14, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v19
+; GISEL-NEXT: v_lshlrev_b16_e32 v0, 15, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v19
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v0
+; GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GISEL-NEXT: v_lshl_or_b32 v10, v0, 16, v0
+; GISEL-NEXT: v_or3_b32 v8, v1, v2, 1
+; GISEL-NEXT: v_or3_b32 v9, v0, v2, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x433
+; GISEL-NEXT: v_mov_b32_e32 v1, 0
+; GISEL-NEXT: v_mov_b32_e32 v2, 0xfffff
+; GISEL-NEXT: v_mov_b32_e32 v3, 0x100000
+; GISEL-NEXT: v_cmp_ge_u64_e32 vcc, v[6:7], v[0:1]
+; GISEL-NEXT: v_and_or_b32 v5, v5, v2, v3
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB0_3
+; GISEL-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; GISEL-NEXT: v_add_u32_e32 v13, 0xfffffbcd, v6
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v13, v[4:5]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v13
+; GISEL-NEXT: v_sub_u32_e32 v2, 64, v13
+; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v0, vcc
+; GISEL-NEXT: v_add_u32_e32 v11, 0xfffffb8d, v6
+; GISEL-NEXT: v_lshrrev_b64 v[6:7], v2, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v14, v10, 0
+; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v1, vcc
+; GISEL-NEXT: v_lshlrev_b64 v[4:5], v11, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v15, v9, v[2:3]
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v13
+; GISEL-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v14, v8, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v4, v8, v[11:12]
+; GISEL-NEXT: v_mul_lo_u32 v6, v15, v10
+; GISEL-NEXT: v_mul_lo_u32 v10, v14, v10
+; GISEL-NEXT: v_mad_u64_u32 v[11:12], s[6:7], v14, v9, v[1:2]
+; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v15, v8, v[11:12]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[8:9], v3, v10, s[8:9]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[6:7], v3, v6, s[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[10:11], s[6:7], v4, v9, v[3:4]
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v7, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v3, 0, s[4:5]
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: ; implicit-def: $vgpr9
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v8, v[10:11]
+; GISEL-NEXT: ; implicit-def: $vgpr4_vgpr5
+; GISEL-NEXT: ; implicit-def: $vgpr8
+; GISEL-NEXT: ; implicit-def: $vgpr10
+; GISEL-NEXT: .LBB0_3: ; %Flow
+; GISEL-NEXT: s_andn2_saveexec_b64 s[6:7], s[12:13]
+; GISEL-NEXT: s_cbranch_execz .LBB0_5
+; GISEL-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; GISEL-NEXT: v_sub_co_u32_e32 v0, vcc, 0x433, v6
+; GISEL-NEXT: v_lshrrev_b64 v[4:5], v0, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v4, v10, 0
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v8, 0
+; GISEL-NEXT: v_mul_lo_u32 v11, v5, v10
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v9, v[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[6:7], vcc, v4, v9, v[1:2]
+; GISEL-NEXT: v_mul_lo_u32 v4, v4, v10
+; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v8, v[6:7]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[4:5], v3, v4, s[4:5]
+; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v11, vcc
+; GISEL-NEXT: .LBB0_5: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
+; GISEL-NEXT: .LBB0_6: ; %fp-to-i-cleanup
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%cvt = fptosi double %x to i128
ret i128 %cvt
}
@@ -214,85 +352,217 @@ define i128 @fptoui_f64_to_i128(double %x) {
}
define i128 @fptosi_f32_to_i128(float %x) {
-; GCN-LABEL: fptosi_f32_to_i128:
-; GCN: ; %bb.0: ; %fp-to-i-entry
-; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v4, v0
-; GCN-NEXT: v_bfe_u32 v9, v4, 23, 8
-; GCN-NEXT: s_movk_i32 s4, 0x7e
-; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: v_cmp_lt_u32_e32 vcc, s4, v9
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
-; GCN-NEXT: s_cbranch_execz .LBB2_6
-; GCN-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
-; GCN-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GCN-NEXT: v_and_b32_e32 v0, 0x7fffff, v4
-; GCN-NEXT: s_movk_i32 s4, 0x95
-; GCN-NEXT: v_ashrrev_i32_e32 v7, 31, v5
-; GCN-NEXT: v_or_b32_e32 v8, 1, v5
-; GCN-NEXT: v_or_b32_e32 v1, 0x800000, v0
-; GCN-NEXT: v_cmp_lt_u32_e32 vcc, s4, v9
-; GCN-NEXT: ; implicit-def: $vgpr2_vgpr3
-; GCN-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; GCN-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; GCN-NEXT: s_cbranch_execz .LBB2_3
-; GCN-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_sub_u32_e32 v3, 0xd6, v9
-; GCN-NEXT: v_add_u32_e32 v6, 0xffffff2a, v9
-; GCN-NEXT: v_add_u32_e32 v0, 0xffffff6a, v9
-; GCN-NEXT: v_lshrrev_b64 v[3:4], v3, v[1:2]
-; GCN-NEXT: v_lshlrev_b64 v[9:10], v6, v[1:2]
-; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0
-; GCN-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc
-; GCN-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
-; GCN-NEXT: v_cndmask_b32_e64 v6, 0, v4, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v9, v9, v3, vcc
-; GCN-NEXT: v_lshlrev_b64 v[3:4], v0, v[1:2]
-; GCN-NEXT: v_cndmask_b32_e64 v9, 0, v9, s[4:5]
-; GCN-NEXT: v_cndmask_b32_e32 v11, 0, v3, vcc
-; GCN-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v8, 0
-; GCN-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
-; GCN-NEXT: v_mul_lo_u32 v12, v7, v9
-; GCN-NEXT: v_mul_lo_u32 v6, v8, v6
-; GCN-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v8, v[1:2]
-; GCN-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v8, v9, 0
-; GCN-NEXT: v_mov_b32_e32 v1, v3
-; GCN-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v7, v[1:2]
-; GCN-NEXT: v_add3_u32 v10, v10, v6, v12
-; GCN-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v11, v[9:10]
-; GCN-NEXT: v_mul_lo_u32 v6, v5, v13
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
-; GCN-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
-; GCN-NEXT: v_mul_lo_u32 v5, v5, v11
-; GCN-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v7, v[2:3]
-; GCN-NEXT: ; implicit-def: $vgpr7
-; GCN-NEXT: v_add3_u32 v4, v5, v9, v6
-; GCN-NEXT: v_add_co_u32_e32 v2, vcc, v2, v8
-; GCN-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GCN-NEXT: ; implicit-def: $vgpr9
-; GCN-NEXT: ; implicit-def: $vgpr8
-; GCN-NEXT: ; implicit-def: $vgpr5_vgpr6
-; GCN-NEXT: .LBB2_3: ; %Flow
-; GCN-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
-; GCN-NEXT: s_cbranch_execz .LBB2_5
-; GCN-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
-; GCN-NEXT: v_sub_u32_e32 v0, 0x96, v9
-; GCN-NEXT: v_lshrrev_b32_e32 v3, v0, v1
-; GCN-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v3, v8, 0
-; GCN-NEXT: v_mov_b32_e32 v2, 0
-; GCN-NEXT: v_mov_b32_e32 v8, v2
-; GCN-NEXT: v_mad_u64_u32 v[6:7], s[8:9], v3, v7, v[1:2]
-; GCN-NEXT: v_mad_i64_i32 v[2:3], s[8:9], v5, v3, v[7:8]
-; GCN-NEXT: v_mov_b32_e32 v1, v6
-; GCN-NEXT: .LBB2_5: ; %Flow1
-; GCN-NEXT: s_or_b64 exec, exec, s[4:5]
-; GCN-NEXT: .LBB2_6: ; %fp-to-i-cleanup
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
-; GCN-NEXT: s_setpc_b64 s[30:31]
+; SDAG-LABEL: fptosi_f32_to_i128:
+; SDAG: ; %bb.0: ; %fp-to-i-entry
+; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-NEXT: v_mov_b32_e32 v4, v0
+; SDAG-NEXT: v_bfe_u32 v9, v4, 23, 8
+; SDAG-NEXT: s_movk_i32 s4, 0x7e
+; SDAG-NEXT: v_mov_b32_e32 v0, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
+; SDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v9
+; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SDAG-NEXT: s_cbranch_execz .LBB2_6
+; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; SDAG-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v4
+; SDAG-NEXT: s_movk_i32 s4, 0x95
+; SDAG-NEXT: v_ashrrev_i32_e32 v7, 31, v5
+; SDAG-NEXT: v_or_b32_e32 v8, 1, v5
+; SDAG-NEXT: v_or_b32_e32 v1, 0x800000, v0
+; SDAG-NEXT: v_cmp_lt_u32_e32 vcc, s4, v9
+; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
+; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
+; SDAG-NEXT: s_cbranch_execz .LBB2_3
+; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_sub_u32_e32 v3, 0xd6, v9
+; SDAG-NEXT: v_add_u32_e32 v6, 0xffffff2a, v9
+; SDAG-NEXT: v_add_u32_e32 v0, 0xffffff6a, v9
+; SDAG-NEXT: v_lshrrev_b64 v[3:4], v3, v[1:2]
+; SDAG-NEXT: v_lshlrev_b64 v[9:10], v6, v[1:2]
+; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v0
+; SDAG-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc
+; SDAG-NEXT: v_cmp_ne_u32_e64 s[4:5], 0, v0
+; SDAG-NEXT: v_cndmask_b32_e64 v6, 0, v4, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v3, vcc
+; SDAG-NEXT: v_lshlrev_b64 v[3:4], v0, v[1:2]
+; SDAG-NEXT: v_cndmask_b32_e64 v9, 0, v9, s[4:5]
+; SDAG-NEXT: v_cndmask_b32_e32 v11, 0, v3, vcc
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v11, v8, 0
+; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
+; SDAG-NEXT: v_mul_lo_u32 v12, v7, v9
+; SDAG-NEXT: v_mul_lo_u32 v6, v8, v6
+; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v8, v[1:2]
+; SDAG-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v8, v9, 0
+; SDAG-NEXT: v_mov_b32_e32 v1, v3
+; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v11, v7, v[1:2]
+; SDAG-NEXT: v_add3_u32 v10, v10, v6, v12
+; SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v5, v11, v[9:10]
+; SDAG-NEXT: v_mul_lo_u32 v6, v5, v13
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
+; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, vcc
+; SDAG-NEXT: v_mul_lo_u32 v5, v5, v11
+; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v7, v[2:3]
+; SDAG-NEXT: ; implicit-def: $vgpr7
+; SDAG-NEXT: v_add3_u32 v4, v5, v9, v6
+; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v8
+; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
+; SDAG-NEXT: ; implicit-def: $vgpr9
+; SDAG-NEXT: ; implicit-def: $vgpr8
+; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6
+; SDAG-NEXT: .LBB2_3: ; %Flow
+; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[8:9]
+; SDAG-NEXT: s_cbranch_execz .LBB2_5
+; SDAG-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; SDAG-NEXT: v_sub_u32_e32 v0, 0x96, v9
+; SDAG-NEXT: v_lshrrev_b32_e32 v3, v0, v1
+; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[8:9], v3, v8, 0
+; SDAG-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-NEXT: v_mov_b32_e32 v8, v2
+; SDAG-NEXT: v_mad_u64_u32 v[6:7], s[8:9], v3, v7, v[1:2]
+; SDAG-NEXT: v_mad_i64_i32 v[2:3], s[8:9], v5, v3, v[7:8]
+; SDAG-NEXT: v_mov_b32_e32 v1, v6
+; SDAG-NEXT: .LBB2_5: ; %Flow1
+; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
+; SDAG-NEXT: .LBB2_6: ; %fp-to-i-cleanup
+; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
+; SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GISEL-LABEL: fptosi_f32_to_i128:
+; GISEL: ; %bb.0: ; %fp-to-i-entry
+; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-NEXT: v_mov_b32_e32 v4, v0
+; GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GISEL-NEXT: v_bfe_u32 v6, v4, 23, 8
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x7f
+; GISEL-NEXT: s_mov_b64 s[6:7], s[4:5]
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v6, v0
+; GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GISEL-NEXT: v_mov_b32_e32 v2, s6
+; GISEL-NEXT: v_mov_b32_e32 v3, s7
+; GISEL-NEXT: s_and_saveexec_b64 s[10:11], vcc
+; GISEL-NEXT: s_cbranch_execz .LBB2_6
+; GISEL-NEXT: ; %bb.1: ; %fp-to-i-if-check.exp.size
+; GISEL-NEXT: v_cmp_ge_i32_e32 vcc, -1, v4
+; GISEL-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
+; GISEL-NEXT: v_and_b32_e32 v0, 1, v0
+; GISEL-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc
+; GISEL-NEXT: v_lshlrev_b16_e32 v2, 1, v0
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GISEL-NEXT: v_lshlrev_b16_e32 v3, 2, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v0, v2
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GISEL-NEXT: v_lshlrev_b16_e32 v7, 3, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v7
+; GISEL-NEXT: v_lshlrev_b16_e32 v8, 4, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v7
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v8
+; GISEL-NEXT: v_lshlrev_b16_e32 v9, 5, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v8
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v9
+; GISEL-NEXT: v_lshlrev_b16_e32 v10, 6, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v9
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v10
+; GISEL-NEXT: v_lshlrev_b16_e32 v11, 7, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v10
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v11
+; GISEL-NEXT: v_lshlrev_b16_e32 v12, 8, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v11
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v12
+; GISEL-NEXT: v_lshlrev_b16_e32 v13, 9, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v12
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v13
+; GISEL-NEXT: v_lshlrev_b16_e32 v14, 10, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v13
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v14
+; GISEL-NEXT: v_lshlrev_b16_e32 v15, 11, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v14
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v15
+; GISEL-NEXT: v_lshlrev_b16_e32 v16, 12, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v15
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v16
+; GISEL-NEXT: v_lshlrev_b16_e32 v17, 13, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v16
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v17
+; GISEL-NEXT: v_lshlrev_b16_e32 v18, 14, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v17
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v18
+; GISEL-NEXT: v_lshlrev_b16_e32 v0, 15, v0
+; GISEL-NEXT: v_or_b32_e32 v2, v2, v18
+; GISEL-NEXT: v_or_b32_e32 v1, v1, v0
+; GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v0
+; GISEL-NEXT: v_lshl_or_b32 v9, v0, 16, v0
+; GISEL-NEXT: v_or3_b32 v7, v1, v2, 1
+; GISEL-NEXT: v_or3_b32 v8, v0, v2, 0
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x7fffff
+; GISEL-NEXT: v_mov_b32_e32 v1, 0x800000
+; GISEL-NEXT: v_and_or_b32 v4, v4, v0, v1
+; GISEL-NEXT: v_mov_b32_e32 v0, 0x96
+; GISEL-NEXT: v_mov_b32_e32 v5, 0
+; GISEL-NEXT: v_cmp_ge_u32_e32 vcc, v6, v0
+; GISEL-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3
+; GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[4:5]
+; GISEL-NEXT: s_cbranch_execz .LBB2_3
+; GISEL-NEXT: ; %bb.2: ; %fp-to-i-if-exp.large
+; GISEL-NEXT: v_add_u32_e32 v14, 0xffffff6a, v6
+; GISEL-NEXT: v_lshlrev_b64 v[0:1], v14, v[4:5]
+; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v14
+; GISEL-NEXT: v_sub_u32_e32 v2, 64, v14
+; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v0, vcc
+; GISEL-NEXT: v_lshrrev_b64 v[10:11], v2, v[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v15, v9, 0
+; GISEL-NEXT: v_add_u32_e32 v6, 0xffffff2a, v6
+; GISEL-NEXT: v_lshlrev_b64 v[4:5], v6, v[4:5]
+; GISEL-NEXT: v_cndmask_b32_e32 v6, 0, v1, vcc
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v6, v8, v[2:3]
+; GISEL-NEXT: v_cndmask_b32_e32 v0, v4, v10, vcc
+; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v14
+; GISEL-NEXT: v_cndmask_b32_e64 v4, v0, 0, s[4:5]
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v15, v7, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v4, v7, v[12:13]
+; GISEL-NEXT: v_mul_lo_u32 v10, v6, v9
+; GISEL-NEXT: v_mul_lo_u32 v9, v15, v9
+; GISEL-NEXT: v_mad_u64_u32 v[12:13], s[6:7], v15, v8, v[1:2]
+; GISEL-NEXT: v_mad_u64_u32 v[1:2], s[8:9], v6, v7, v[12:13]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[8:9], v3, v9, s[8:9]
+; GISEL-NEXT: v_addc_co_u32_e64 v3, s[6:7], v3, v10, s[6:7]
+; GISEL-NEXT: v_mad_u64_u32 v[9:10], s[6:7], v4, v8, v[3:4]
+; GISEL-NEXT: v_cndmask_b32_e32 v3, v5, v11, vcc
+; GISEL-NEXT: v_cndmask_b32_e64 v5, v3, 0, s[4:5]
+; GISEL-NEXT: ; implicit-def: $vgpr6
+; GISEL-NEXT: ; implicit-def: $vgpr8
+; GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v7, v[9:10]
+; GISEL-NEXT: ; implicit-def: $vgpr4
+; GISEL-NEXT: ; implicit-def: $vgpr7
+; GISEL-NEXT: ; implicit-def: $vgpr9
+; GISEL-NEXT: .LBB2_3: ; %Flow
+; GISEL-NEXT: s_andn2_saveexec_b64 s[4:5], s[12:13]
+; GISEL-NEXT: s_cbranch_execz .LBB2_5
+; GISEL-NEXT: ; %bb.4: ; %fp-to-i-if-exp.small
+; GISEL-NEXT: v_sub_u32_e32 v0, 0x96, v6
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, v0, v4
+; GISEL-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v6, v7, 0
+; GISEL-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v6, v9, 0
+; GISEL-NEXT: v_mul_lo_u32 v7, v6, v9
+; GISEL-NEXT: v_mad_u64_u32 v[4:5], vcc, v6, v8, v[1:2]
+; GISEL-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v7, vcc
+; GISEL-NEXT: v_mov_b32_e32 v1, v4
+; GISEL-NEXT: v_mov_b32_e32 v2, v5
+; GISEL-NEXT: .LBB2_5: ; %Flow1
+; GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
+; GISEL-NEXT: .LBB2_6: ; %fp-to-i-cleanup
+; GISEL-NEXT: s_or_b64 exec, exec, s[10:11]
+; GISEL-NEXT: s_setpc_b64 s[30:31]
%cvt = fptosi float %x to i128
ret i128 %cvt
}
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
index 2622d77e27402..402c10f4e3ab2 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi-sat-vector.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-ISEL
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu9.00 | FileCheck %s --check-prefixes=GFX9
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
@@ -1478,19 +1478,34 @@ define <2 x i8> @test_signed_v2f64_v2i8(<2 x double> %f) {
}
define <2 x i16> @test_signed_v2f64_v2i16(<2 x double> %f) {
-; GFX7-LABEL: test_signed_v2f64_v2i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_i32_f64_e32 v2, v[2:3]
-; GFX7-NEXT: v_cvt_i32_f64_e32 v0, v[0:1]
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX7-NEXT: v_med3_i32 v2, v2, s4, v1
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_v2f64_v2i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v2, v[2:3]
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v0, v[0:1]
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s4, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-ISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_v2f64_v2i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v0, v[0:1]
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v1, v[2:3]
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, 0xffff8000
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v3, v2
+; GFX7-GI-NEXT: v_med3_i32 v1, v1, v3, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_v2f64_v2i16:
; GFX9: ; %bb.0:
@@ -2054,19 +2069,37 @@ define <2 x i8> @test_s_signed_v2f64_v2i8(<2 x double> inreg %f) {
}
define <2 x i16> @test_s_signed_v2f64_v2i16(<2 x double> inreg %f) {
-; GFX7-LABEL: test_s_signed_v2f64_v2i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_i32_f64_e32 v0, s[18:19]
-; GFX7-NEXT: v_cvt_i32_f64_e32 v1, s[16:17]
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v2, 0x7fff
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v2
-; GFX7-NEXT: v_med3_i32 v1, v1, s4, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_signed_v2f64_v2i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v0, s[18:19]
+; GFX7-ISEL-NEXT: v_cvt_i32_f64_e32 v1, s[16:17]
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v2, 0x7fff
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v2
+; GFX7-ISEL-NEXT: v_med3_i32 v1, v1, s4, v2
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_signed_v2f64_v2i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v1, s[18:19]
+; GFX7-GI-NEXT: v_cvt_i32_f64_e32 v0, s[16:17]
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v1
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: s_min_i32 s5, s5, 0x7fff
+; GFX7-GI-NEXT: s_min_i32 s4, s4, 0x7fff
+; GFX7-GI-NEXT: s_max_i32 s5, s5, 0xffff8000
+; GFX7-GI-NEXT: s_max_i32 s4, s4, 0xffff8000
+; GFX7-GI-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX7-GI-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX7-GI-NEXT: s_lshl_b32 s5, s5, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s4, s5
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_signed_v2f64_v2i16:
; GFX9: ; %bb.0:
@@ -2810,32 +2843,61 @@ define <4 x i8> @test_signed_v4f16_v4i8(<4 x half> %f) {
}
define <4 x i16> @test_signed_v4f16_v4i16(<4 x half> %f) {
-; GFX7-LABEL: test_signed_v4f16_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX7-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v4, 0x7fff
-; GFX7-NEXT: v_med3_i32 v2, v2, s4, v4
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v4
-; GFX7-NEXT: v_med3_i32 v1, v1, s4, v4
-; GFX7-NEXT: v_med3_i32 v3, v3, s4, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_v4f16_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s4, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v1, v1, s4, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v3, v3, s4, v4
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_v4f16_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v4, 0x8000
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_med3_i32 v2, v2, v4, s4
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v4, s4
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_med3_i32 v3, v3, v4, s4
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GI-NEXT: v_med3_i32 v1, v1, v4, s4
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GI-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_v4f16_v4i16:
; GFX9: ; %bb.0:
@@ -3685,32 +3747,67 @@ define <4 x i8> @test_s_signed_v4f16_v4i8(<4 x half> inreg %f) {
}
define <4 x i16> @test_s_signed_v4f16_v4i16(<4 x half> inreg %f) {
-; GFX7-LABEL: test_s_signed_v4f16_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s17
-; GFX7-NEXT: s_lshr_b32 s4, s16, 16
-; GFX7-NEXT: s_lshr_b32 s5, s17, 16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, s16
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, s4
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, s5
-; GFX7-NEXT: s_movk_i32 s6, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v3, 0x7fff
-; GFX7-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX7-NEXT: v_med3_i32 v5, v0, s6, v3
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v4
-; GFX7-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX7-NEXT: v_med3_i32 v1, v1, s6, v3
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_med3_i32 v0, v0, s6, v3
-; GFX7-NEXT: v_med3_i32 v2, v2, s6, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_signed_v4f16_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s17
+; GFX7-ISEL-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-ISEL-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, s16
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v4, s4
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, s5
+; GFX7-ISEL-NEXT: s_movk_i32 s6, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v5, v0, s6, v3
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v4
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_med3_i32 v1, v1, s6, v3
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s6, v3
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s6, v3
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v5
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_signed_v4f16_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-GI-NEXT: s_lshr_b32 s5, s16, 16
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, s5
+; GFX7-GI-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, s17
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v4, s5
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, 0x8000
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v1, s4
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v3
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v4
+; GFX7-GI-NEXT: v_med3_i32 v2, v2, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-GI-NEXT: v_med3_i32 v0, v3, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX7-GI-NEXT: s_lshl_b32 s6, s6, 16
+; GFX7-GI-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX7-GI-NEXT: s_or_b32 s5, s5, s6
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s7
+; GFX7-GI-NEXT: s_lshl_b32 s4, s4, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s6, s4
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_signed_v4f16_v4i16:
; GFX9: ; %bb.0:
@@ -4547,52 +4644,103 @@ define <8 x i8> @test_signed_v8f16_v8i8(<8 x half> %f) {
}
define <8 x i16> @test_signed_v8f16_v8i16(<8 x half> %f) {
-; GFX7-LABEL: test_signed_v8f16_v8i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX7-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_i32_f32_e32 v7, v7
-; GFX7-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX7-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX7-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX7-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v8, 0x7fff
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v8
-; GFX7-NEXT: v_med3_i32 v4, v4, s4, v8
-; GFX7-NEXT: v_med3_i32 v2, v2, s4, v8
-; GFX7-NEXT: v_med3_i32 v7, v7, s4, v8
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_med3_i32 v5, v5, s4, v8
-; GFX7-NEXT: v_med3_i32 v1, v1, s4, v8
-; GFX7-NEXT: v_med3_i32 v3, v3, s4, v8
-; GFX7-NEXT: v_med3_i32 v6, v6, s4, v8
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX7-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_v8f16_v8i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v5, v3
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v7, v7
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v5, v5
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v6, v6
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v8, 0x7fff
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v4, v4, s4, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s4, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v7, v7, s4, v8
+; GFX7-ISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v5, v5, s4, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v1, v1, s4, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v3, v3, s4, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v6, v6, s4, v8
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-ISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-ISEL-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v1, v6
+; GFX7-ISEL-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_v8f16_v8i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v5, v5
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v8, 0x8000
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-GI-NEXT: v_med3_i32 v4, v4, v8, s4
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v6, v6
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v8, s4
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GI-NEXT: v_med3_i32 v5, v5, v8, s4
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v7, v7
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_med3_i32 v1, v1, v8, s4
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX7-GI-NEXT: v_med3_i32 v6, v6, v8, s4
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_med3_i32 v2, v2, v8, s4
+; GFX7-GI-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX7-GI-NEXT: v_med3_i32 v7, v7, v8, s4
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_med3_i32 v3, v3, v8, s4
+; GFX7-GI-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX7-GI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_v8f16_v8i16:
; GFX9: ; %bb.0:
@@ -6045,52 +6193,115 @@ define <8 x i8> @test_s_signed_v8f16_v8i8(<8 x half> inreg %f) {
}
define <8 x i16> @test_s_signed_v8f16_v8i16(<8 x half> inreg %f) {
-; GFX7-LABEL: test_s_signed_v8f16_v8i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s19
-; GFX7-NEXT: s_lshr_b32 s4, s16, 16
-; GFX7-NEXT: s_lshr_b32 s5, s17, 16
-; GFX7-NEXT: s_lshr_b32 s6, s18, 16
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: s_lshr_b32 s7, s19, 16
-; GFX7-NEXT: s_movk_i32 s8, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, s18
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, s17
-; GFX7-NEXT: v_med3_i32 v4, v0, s8, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v5, s7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v6, s6
-; GFX7-NEXT: v_cvt_f32_f16_e32 v7, s5
-; GFX7-NEXT: v_cvt_f32_f16_e32 v8, s4
-; GFX7-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX7-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX7-NEXT: v_cvt_i32_f32_e32 v7, v7
-; GFX7-NEXT: v_cvt_i32_f32_e32 v8, v8
-; GFX7-NEXT: v_med3_i32 v2, v2, s8, v1
-; GFX7-NEXT: v_med3_i32 v3, v3, s8, v1
-; GFX7-NEXT: v_med3_i32 v0, v0, s8, v1
-; GFX7-NEXT: v_med3_i32 v5, v5, s8, v1
-; GFX7-NEXT: v_med3_i32 v6, v6, s8, v1
-; GFX7-NEXT: v_med3_i32 v7, v7, s8, v1
-; GFX7-NEXT: v_med3_i32 v1, v8, s8, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v3
-; GFX7-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_lshlrev_b32_e32 v7, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v7
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_signed_v8f16_v8i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s19
+; GFX7-ISEL-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-ISEL-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-ISEL-NEXT: s_lshr_b32 s6, s18, 16
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: s_lshr_b32 s7, s19, 16
+; GFX7-ISEL-NEXT: s_movk_i32 s8, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, s18
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, s17
+; GFX7-ISEL-NEXT: v_med3_i32 v4, v0, s8, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v5, s7
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v6, s6
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v7, s5
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v8, s4
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v5, v5
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v6, v6
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v7, v7
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v8, v8
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s8, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v3, v3, s8, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s8, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v5, v5, s8, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v6, v6, s8, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v7, v7, s8, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v1, v8, s8, v1
+; GFX7-ISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GFX7-ISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX7-ISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX7-ISEL-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v1, v7
+; GFX7-ISEL-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_signed_v8f16_v8i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, s17
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, 0x8000
+; GFX7-GI-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v2, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s8, v0
+; GFX7-GI-NEXT: v_med3_i32 v0, v1, v2, s4
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s5
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s18
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-GI-NEXT: s_lshr_b32 s6, s18, 16
+; GFX7-GI-NEXT: v_med3_i32 v3, v3, v2, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s9, v3
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, s6
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_med3_i32 v1, v1, v2, s4
+; GFX7-GI-NEXT: s_lshr_b32 s7, s19, 16
+; GFX7-GI-NEXT: v_readfirstlane_b32 s6, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s19
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v4, s7
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v2, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v1
+; GFX7-GI-NEXT: v_med3_i32 v1, v3, v2, s4
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v4
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v2, s4
+; GFX7-GI-NEXT: s_and_b32 s8, 0xffff, s8
+; GFX7-GI-NEXT: s_lshl_b32 s5, s5, 16
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX7-GI-NEXT: v_readfirstlane_b32 s10, v1
+; GFX7-GI-NEXT: v_readfirstlane_b32 s11, v0
+; GFX7-GI-NEXT: v_med3_i32 v0, v3, v2, s4
+; GFX7-GI-NEXT: s_or_b32 s5, s8, s5
+; GFX7-GI-NEXT: s_and_b32 s8, 0xffff, s9
+; GFX7-GI-NEXT: s_lshl_b32 s6, s6, 16
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: s_or_b32 s6, s8, s6
+; GFX7-GI-NEXT: s_and_b32 s8, 0xffff, s10
+; GFX7-GI-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX7-GI-NEXT: s_lshl_b32 s8, s8, 16
+; GFX7-GI-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX7-GI-NEXT: s_or_b32 s7, s7, s8
+; GFX7-GI-NEXT: s_and_b32 s8, 0xffff, s11
+; GFX7-GI-NEXT: s_lshl_b32 s4, s4, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s8, s4
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, s6
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, s7
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_signed_v8f16_v8i16:
; GFX9: ; %bb.0:
@@ -6477,26 +6688,49 @@ define <8 x i64> @test_s_signed_v8f16_v8i64(<8 x half> inreg %f) {
;
define <4 x i16> @test_signed_v4f32_v4i16(<4 x float> %f) {
-; GFX7-LABEL: test_signed_v4f32_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_i32_f32_e32 v1, v1
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX7-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v4, 0x7fff
-; GFX7-NEXT: v_med3_i32 v1, v1, s4, v4
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v4
-; GFX7-NEXT: v_med3_i32 v3, v3, s4, v4
-; GFX7-NEXT: v_med3_i32 v2, v2, s4, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_signed_v4f32_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX7-ISEL-NEXT: v_med3_i32 v1, v1, s4, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v3, v3, s4, v4
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s4, v4
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_signed_v4f32_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, v3
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v4, 0x8000
+; GFX7-GI-NEXT: v_med3_i32 v1, v1, v4, s4
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v4, s4
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_med3_i32 v2, v2, v4, s4
+; GFX7-GI-NEXT: v_med3_i32 v3, v3, v4, s4
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GI-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_signed_v4f32_v4i16:
; GFX9: ; %bb.0:
@@ -6538,26 +6772,55 @@ define <4 x i16> @test_signed_v4f32_v4i16(<4 x float> %f) {
}
define <4 x i16> @test_s_signed_v4f32_v4i16(<4 x float> inreg %f) {
-; GFX7-LABEL: test_s_signed_v4f32_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, s19
-; GFX7-NEXT: s_movk_i32 s4, 0x8000
-; GFX7-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX7-NEXT: v_cvt_i32_f32_e32 v2, s18
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v1
-; GFX7-NEXT: v_cvt_i32_f32_e32 v3, s17
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GFX7-NEXT: v_cvt_i32_f32_e32 v0, s16
-; GFX7-NEXT: v_med3_i32 v2, v2, s4, v1
-; GFX7-NEXT: v_med3_i32 v3, v3, s4, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_med3_i32 v0, v0, s4, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_signed_v4f32_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, s19
+; GFX7-ISEL-NEXT: s_movk_i32 s4, 0x8000
+; GFX7-ISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v2, s18
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v3, s17
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v0
+; GFX7-ISEL-NEXT: v_cvt_i32_f32_e32 v0, s16
+; GFX7-ISEL-NEXT: v_med3_i32 v2, v2, s4, v1
+; GFX7-ISEL-NEXT: v_med3_i32 v3, v3, s4, v1
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_med3_i32 v0, v0, s4, v1
+; GFX7-ISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_signed_v4f32_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, s16
+; GFX7-GI-NEXT: s_movk_i32 s4, 0x7fff
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, 0x8000
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v2, s17
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v0, s18
+; GFX7-GI-NEXT: v_cvt_i32_f32_e32 v3, s19
+; GFX7-GI-NEXT: v_med3_i32 v2, v2, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s6, v2
+; GFX7-GI-NEXT: v_med3_i32 v0, v0, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-GI-NEXT: v_med3_i32 v0, v3, v1, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX7-GI-NEXT: s_lshl_b32 s6, s6, 16
+; GFX7-GI-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX7-GI-NEXT: s_or_b32 s5, s5, s6
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s7
+; GFX7-GI-NEXT: s_lshl_b32 s4, s4, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s6, s4
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_signed_v4f32_v4i16:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
index 4689a8e695f87..69eec3819ec4f 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui-sat-vector.ll
@@ -1,6 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-ISEL
-; RUN: llc < %s -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
+; RUN: llc < %s -global-isel=1 -mtriple=amdgpu7.00 | FileCheck %s --check-prefixes=GFX7,GFX7-GI
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu9.00 | FileCheck %s --check-prefixes=GFX9
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-FAKE16
; RUN: llc < %s -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 | FileCheck %s --check-prefixes=GFX11,GFX11-TRUE16
@@ -1946,16 +1946,30 @@ define <2 x i8> @test_s_unsigned_v2f64_v2i8(<2 x double> inreg %f) {
}
define <2 x i16> @test_s_unsigned_v2f64_v2i16(<2 x double> inreg %f) {
-; GFX7-LABEL: test_s_unsigned_v2f64_v2i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_u32_f64_e32 v1, s[18:19]
-; GFX7-NEXT: v_cvt_u32_f64_e32 v0, s[16:17]
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_unsigned_v2f64_v2i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v1, s[18:19]
+; GFX7-ISEL-NEXT: v_cvt_u32_f64_e32 v0, s[16:17]
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_unsigned_v2f64_v2i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v1, s[18:19]
+; GFX7-GI-NEXT: v_cvt_u32_f64_e32 v0, s[16:17]
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v1
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: s_min_u32 s5, s5, 0xffff
+; GFX7-GI-NEXT: s_min_u32 s4, s4, 0xffff
+; GFX7-GI-NEXT: s_lshl_b32 s5, s5, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s4, s5
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_unsigned_v2f64_v2i16:
; GFX9: ; %bb.0:
@@ -2606,28 +2620,55 @@ define <4 x i8> @test_unsigned_v4f16_v4i8(<4 x half> %f) {
}
define <4 x i16> @test_unsigned_v4f16_v4i16(<4 x half> %f) {
-; GFX7-LABEL: test_unsigned_v4f16_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX7-NEXT: v_min_u32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_min_u32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_unsigned_v4f16_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, v1
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_unsigned_v4f16_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GI-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_unsigned_v4f16_v4i16:
; GFX9: ; %bb.0:
@@ -3416,28 +3457,61 @@ define <4 x i8> @test_s_unsigned_v4f16_v4i8(<4 x half> inreg %f) {
}
define <4 x i16> @test_s_unsigned_v4f16_v4i16(<4 x half> inreg %f) {
-; GFX7-LABEL: test_s_unsigned_v4f16_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s17
-; GFX7-NEXT: s_lshr_b32 s4, s16, 16
-; GFX7-NEXT: s_lshr_b32 s5, s17, 16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, s5
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, s16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, s4
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX7-NEXT: v_min_u32_e32 v4, 0xffff, v0
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v2
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v3
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_min_u32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_unsigned_v4f16_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s17
+; GFX7-ISEL-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-ISEL-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, s5
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, s16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, s4
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v4, 0xffff, v0
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v3
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_unsigned_v4f16_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-GI-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; GFX7-GI-NEXT: s_lshr_b32 s4, s17, 16
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, s17
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, s4
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v2
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v3
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v2
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX7-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-GI-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX7-GI-NEXT: s_lshl_b32 s5, s5, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s4, s5
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s6
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s7
+; GFX7-GI-NEXT: s_lshl_b32 s6, s6, 16
+; GFX7-GI-NEXT: s_or_b32 s5, s5, s6
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_unsigned_v4f16_v4i16:
; GFX9: ; %bb.0:
@@ -4231,46 +4305,95 @@ define <8 x i8> @test_unsigned_v8f16_v8i8(<8 x half> %f) {
}
define <8 x i16> @test_unsigned_v8f16_v8i16(<8 x half> %f) {
-; GFX7-LABEL: test_unsigned_v8f16_v8i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v3
-; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX7-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX7-NEXT: v_cvt_f32_f16_e32 v7, v7
-; GFX7-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v4, v4
-; GFX7-NEXT: v_cvt_u32_f32_e32 v5, v5
-; GFX7-NEXT: v_cvt_u32_f32_e32 v7, v7
-; GFX7-NEXT: v_cvt_u32_f32_e32 v6, v6
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX7-NEXT: v_min_u32_e32 v4, 0xffff, v4
-; GFX7-NEXT: v_min_u32_e32 v5, 0xffff, v5
-; GFX7-NEXT: v_min_u32_e32 v7, 0xffff, v7
-; GFX7-NEXT: v_min_u32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_min_u32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX7-NEXT: v_min_u32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_unsigned_v8f16_v8i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v3
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX7-ISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v5, v5
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v7, v7
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v4, 0xffff, v4
+; GFX7-ISEL-NEXT: v_min_u32_e32 v5, 0xffff, v5
+; GFX7-ISEL-NEXT: v_min_u32_e32 v7, 0xffff, v7
+; GFX7-ISEL-NEXT: v_min_u32_e32 v6, 0xffff, v6
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-ISEL-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-ISEL-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v7
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v1, v6
+; GFX7-ISEL-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-ISEL-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_unsigned_v8f16_v8i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v4, v4
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX7-GI-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v5, v5
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX7-GI-NEXT: v_min_u32_e32 v4, 0xffff, v4
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v6, v6
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX7-GI-NEXT: v_min_u32_e32 v5, 0xffff, v5
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v7, v7
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v5
+; GFX7-GI-NEXT: v_min_u32_e32 v6, 0xffff, v6
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX7-GI-NEXT: v_min_u32_e32 v7, 0xffff, v7
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-GI-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-GI-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GFX7-GI-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX7-GI-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_unsigned_v8f16_v8i16:
; GFX9: ; %bb.0:
@@ -5604,46 +5727,107 @@ define <8 x i8> @test_s_unsigned_v8f16_v8i8(<8 x half> inreg %f) {
}
define <8 x i16> @test_s_unsigned_v8f16_v8i16(<8 x half> inreg %f) {
-; GFX7-LABEL: test_s_unsigned_v8f16_v8i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: s_lshr_b32 s5, s19, 16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s19
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, s5
-; GFX7-NEXT: s_lshr_b32 s6, s18, 16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, s18
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v3, s6
-; GFX7-NEXT: s_lshr_b32 s4, s16, 16
-; GFX7-NEXT: v_min_u32_e32 v4, 0xffff, v0
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v2
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, s17
-; GFX7-NEXT: s_lshr_b32 s5, s17, 16
-; GFX7-NEXT: v_min_u32_e32 v5, 0xffff, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, s5
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v6, s16
-; GFX7-NEXT: v_cvt_f32_f16_e32 v7, s4
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX7-NEXT: v_min_u32_e32 v8, 0xffff, v0
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v6
-; GFX7-NEXT: v_cvt_u32_f32_e32 v6, v7
-; GFX7-NEXT: v_min_u32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_min_u32_e32 v6, 0xffff, v6
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v8, v1
-; GFX7-NEXT: v_or_b32_e32 v2, v5, v2
-; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_unsigned_v8f16_v8i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: s_lshr_b32 s5, s19, 16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s19
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, s5
+; GFX7-ISEL-NEXT: s_lshr_b32 s6, s18, 16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v2, s18
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v3, s6
+; GFX7-ISEL-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-ISEL-NEXT: v_min_u32_e32 v4, 0xffff, v0
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v1
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v2
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v3
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v0, s17
+; GFX7-ISEL-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-ISEL-NEXT: v_min_u32_e32 v5, 0xffff, v1
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v1, s5
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v6, s16
+; GFX7-ISEL-NEXT: v_cvt_f32_f16_e32 v7, s4
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v8, 0xffff, v0
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v6
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v6, v7
+; GFX7-ISEL-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v6, 0xffff, v6
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v6
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v8, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v2, v5, v2
+; GFX7-ISEL-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_unsigned_v8f16_v8i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s16
+; GFX7-GI-NEXT: s_lshr_b32 s4, s16, 16
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, s17
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-GI-NEXT: s_lshr_b32 s5, s17, 16
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s5
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v0
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v0, s18
+; GFX7-GI-NEXT: s_lshr_b32 s4, s18, 16
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_readfirstlane_b32 s8, v2
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v2, s4
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: s_lshr_b32 s6, s19, 16
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v1, s19
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_f32_f16_e32 v3, s6
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v2
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v3
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: s_and_b32 s7, 0xffff, s7
+; GFX7-GI-NEXT: s_lshl_b32 s5, s5, 16
+; GFX7-GI-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX7-GI-NEXT: v_readfirstlane_b32 s9, v1
+; GFX7-GI-NEXT: v_readfirstlane_b32 s10, v0
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v2
+; GFX7-GI-NEXT: s_or_b32 s5, s7, s5
+; GFX7-GI-NEXT: s_and_b32 s7, 0xffff, s8
+; GFX7-GI-NEXT: s_lshl_b32 s4, s4, 16
+; GFX7-GI-NEXT: v_readfirstlane_b32 s11, v0
+; GFX7-GI-NEXT: s_or_b32 s4, s7, s4
+; GFX7-GI-NEXT: s_and_b32 s7, 0xffff, s9
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s6
+; GFX7-GI-NEXT: s_lshl_b32 s7, s7, 16
+; GFX7-GI-NEXT: s_and_b32 s8, 0xffff, s11
+; GFX7-GI-NEXT: s_or_b32 s6, s6, s7
+; GFX7-GI-NEXT: s_and_b32 s7, 0xffff, s10
+; GFX7-GI-NEXT: s_lshl_b32 s8, s8, 16
+; GFX7-GI-NEXT: s_or_b32 s7, s7, s8
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, s4
+; GFX7-GI-NEXT: v_mov_b32_e32 v2, s6
+; GFX7-GI-NEXT: v_mov_b32_e32 v3, s7
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_unsigned_v8f16_v8i16:
; GFX9: ; %bb.0:
@@ -6021,22 +6205,43 @@ define <8 x i64> @test_s_unsigned_v8f16_v8i64(<8 x half> inreg %f) {
;
define <4 x i16> @test_unsigned_v4f32_v4i16(<4 x float> %f) {
-; GFX7-LABEL: test_unsigned_v4f32_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, v3
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX7-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX7-NEXT: v_min_u32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_min_u32_e32 v2, 0xffff, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v3
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_unsigned_v4f32_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-ISEL-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v2, v3
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_unsigned_v4f32_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, v1
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v3, v3
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_min_u32_e32 v2, 0xffff, v2
+; GFX7-GI-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-GI-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-GI-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GI-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX7-GI-NEXT: v_and_b32_e32 v2, 0xffff, v3
+; GFX7-GI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GI-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_unsigned_v4f32_v4i16:
; GFX9: ; %bb.0:
@@ -6076,22 +6281,49 @@ define <4 x i16> @test_unsigned_v4f32_v4i16(<4 x float> %f) {
}
define <4 x i16> @test_s_unsigned_v4f32_v4i16(<4 x float> inreg %f) {
-; GFX7-LABEL: test_s_unsigned_v4f32_v4i16:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, s18
-; GFX7-NEXT: v_cvt_u32_f32_e32 v1, s19
-; GFX7-NEXT: v_cvt_u32_f32_e32 v3, s17
-; GFX7-NEXT: v_min_u32_e32 v2, 0xffff, v0
-; GFX7-NEXT: v_cvt_u32_f32_e32 v0, s16
-; GFX7-NEXT: v_min_u32_e32 v1, 0xffff, v1
-; GFX7-NEXT: v_min_u32_e32 v3, 0xffff, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_min_u32_e32 v0, 0xffff, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-ISEL-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX7-ISEL: ; %bb.0:
+; GFX7-ISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, s18
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v1, s19
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v3, s17
+; GFX7-ISEL-NEXT: v_min_u32_e32 v2, 0xffff, v0
+; GFX7-ISEL-NEXT: v_cvt_u32_f32_e32 v0, s16
+; GFX7-ISEL-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v3, 0xffff, v3
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-ISEL-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-ISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX7-ISEL-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-ISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX7-GI-LABEL: test_s_unsigned_v4f32_v4i16:
+; GFX7-GI: ; %bb.0:
+; GFX7-GI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, s16
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v1, s17
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v2, s19
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_readfirstlane_b32 s4, v0
+; GFX7-GI-NEXT: v_cvt_u32_f32_e32 v0, s18
+; GFX7-GI-NEXT: v_min_u32_e32 v1, 0xffff, v1
+; GFX7-GI-NEXT: v_readfirstlane_b32 s5, v1
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s5
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v0
+; GFX7-GI-NEXT: v_readfirstlane_b32 s6, v0
+; GFX7-GI-NEXT: v_min_u32_e32 v0, 0xffff, v2
+; GFX7-GI-NEXT: v_readfirstlane_b32 s7, v0
+; GFX7-GI-NEXT: s_and_b32 s4, 0xffff, s4
+; GFX7-GI-NEXT: s_lshl_b32 s5, s5, 16
+; GFX7-GI-NEXT: s_or_b32 s4, s4, s5
+; GFX7-GI-NEXT: s_and_b32 s5, 0xffff, s6
+; GFX7-GI-NEXT: s_and_b32 s6, 0xffff, s7
+; GFX7-GI-NEXT: s_lshl_b32 s6, s6, 16
+; GFX7-GI-NEXT: s_or_b32 s5, s5, s6
+; GFX7-GI-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-GI-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-GI-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_s_unsigned_v4f32_v4i16:
; GFX9: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
index 871c6e4e4458f..aed733e40ef2a 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.f16.ll
@@ -1948,22 +1948,21 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; SI-GISEL-LABEL: fptrunc_v2f32_to_v2f16:
; SI-GISEL: ; %bb.0: ; %entry
; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; SI-GISEL-NEXT: s_mov_b32 s6, -1
-; SI-GISEL-NEXT: s_mov_b32 s10, s6
-; SI-GISEL-NEXT: s_mov_b32 s11, s7
; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: s_mov_b32 s8, s2
-; SI-GISEL-NEXT: s_mov_b32 s9, s3
-; SI-GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; SI-GISEL-NEXT: s_mov_b32 s4, s0
-; SI-GISEL-NEXT: s_mov_b32 s5, s1
-; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-GISEL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; SI-GISEL-NEXT: s_mov_b32 s2, -1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; SI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; SI-GISEL-NEXT: s_bfe_u32 s4, s4, 0x100000
+; SI-GISEL-NEXT: s_bfe_u32 s3, s3, 0x100000
+; SI-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; SI-GISEL-NEXT: s_or_b32 s3, s3, s4
+; SI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: fptrunc_v2f32_to_v2f16:
@@ -1989,21 +1988,21 @@ define amdgpu_kernel void @fptrunc_v2f32_to_v2f16(
; VI-GISEL-LABEL: fptrunc_v2f32_to_v2f16:
; VI-GISEL: ; %bb.0: ; %entry
; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: s_mov_b32 s7, 0xf000
-; VI-GISEL-NEXT: s_mov_b32 s6, -1
-; VI-GISEL-NEXT: s_mov_b32 s10, s6
-; VI-GISEL-NEXT: s_mov_b32 s11, s7
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: s_mov_b32 s8, s2
-; VI-GISEL-NEXT: s_mov_b32 s9, s3
-; VI-GISEL-NEXT: buffer_load_dwordx2 v[0:1], off, s[8:11], 0
-; VI-GISEL-NEXT: s_mov_b32 s4, s0
-; VI-GISEL-NEXT: s_mov_b32 s5, s1
-; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-GISEL-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-GISEL-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, s3
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, s2
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; VI-GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; VI-GISEL-NEXT: s_bfe_u32 s4, s4, 0x100000
+; VI-GISEL-NEXT: s_bfe_u32 s3, s3, 0x100000
+; VI-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; VI-GISEL-NEXT: s_or_b32 s3, s3, s4
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: fptrunc_v2f32_to_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/fptrunc.ll b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
index 9705bb72726e1..daeea63e72282 100644
--- a/llvm/test/CodeGen/AMDGPU/fptrunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptrunc.ll
@@ -2203,9 +2203,11 @@ define <2 x half> @fabs_fptrunc_v2f32_to_v2f16(<2 x float> %x) {
; VI-GISEL-LABEL: fabs_fptrunc_v2f32_to_v2f16:
; VI-GISEL: ; %bb.0:
; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-GISEL-NEXT: v_cvt_f16_f32_sdwa v1, |v1| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v1, |v1|
; VI-GISEL-NEXT: v_cvt_f16_f32_e64 v0, |v0|
-; VI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: fabs_fptrunc_v2f32_to_v2f16:
diff --git a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
index a754da7d00499..fd54b36791e86 100644
--- a/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
+++ b/llvm/test/CodeGen/AMDGPU/global-load-xcnt.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GCN-SDAG,GCN-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL,GCN-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL,GCN-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GCN-SDAG,GCN-SDAG-REAL16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL,GCN-GISEL-REAL16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.50-amd-amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GCN-GISEL,GCN-GISEL-REAL16 %s
; Test S_WAIT_XCNT insertion for global_load/store clauses.
; Introduced additional operations in between the clauses to have the register dependency
@@ -27,23 +27,23 @@ define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-FAKE16-NEXT: global_store_b32 v[8:9], v0, off
; GCN-SDAG-FAKE16-NEXT: s_set_pc_i64 s[30:31]
;
-; GCN-GISEL-FAKE16-LABEL: test_i8load_v4i8store:
-; GCN-GISEL-FAKE16: ; %bb.0:
-; GCN-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GCN-GISEL-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: global_load_u8 v6, v[4:5], off
-; GCN-GISEL-FAKE16-NEXT: global_load_u8 v7, v[2:3], off
-; GCN-GISEL-FAKE16-NEXT: global_load_u8 v10, v[0:1], off
-; GCN-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x2
-; GCN-GISEL-FAKE16-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: v_lshlrev_b16 v0, 8, v6
-; GCN-GISEL-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-FAKE16-NEXT: v_perm_b32 v1, v10, v7, 0xc0c0004
-; GCN-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-GISEL-FAKE16-NEXT: v_or_b32_e32 v0, v6, v0
-; GCN-GISEL-FAKE16-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GCN-GISEL-FAKE16-NEXT: global_store_b32 v[8:9], v0, off
-; GCN-GISEL-FAKE16-NEXT: s_set_pc_i64 s[30:31]
+; GCN-GISEL-LABEL: test_i8load_v4i8store:
+; GCN-GISEL: ; %bb.0:
+; GCN-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
+; GCN-GISEL-NEXT: s_wait_kmcnt 0x0
+; GCN-GISEL-NEXT: global_load_u8 v6, v[0:1], off
+; GCN-GISEL-NEXT: global_load_u8 v7, v[2:3], off
+; GCN-GISEL-NEXT: global_load_u8 v10, v[4:5], off
+; GCN-GISEL-NEXT: s_wait_loadcnt 0x1
+; GCN-GISEL-NEXT: s_wait_xcnt 0x2
+; GCN-GISEL-NEXT: v_lshl_or_b32 v0, v7, 8, v6
+; GCN-GISEL-NEXT: s_wait_loadcnt 0x0
+; GCN-GISEL-NEXT: s_wait_xcnt 0x1
+; GCN-GISEL-NEXT: v_dual_lshlrev_b32 v1, 16, v10 :: v_dual_lshlrev_b32 v2, 24, v10
+; GCN-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GCN-GISEL-NEXT: v_or3_b32 v0, v0, v1, v2
+; GCN-GISEL-NEXT: global_store_b32 v[8:9], v0, off
+; GCN-GISEL-NEXT: s_set_pc_i64 s[30:31]
;
; GCN-SDAG-REAL16-LABEL: test_i8load_v4i8store:
; GCN-SDAG-REAL16: ; %bb.0:
@@ -62,24 +62,6 @@ define void @test_i8load_v4i8store(ptr addrspace(1) %ptr_a, ptr addrspace(1) %pt
; GCN-SDAG-REAL16-NEXT: v_or_b16 v1.h, v0.l, v1.h
; GCN-SDAG-REAL16-NEXT: global_store_b32 v[8:9], v1, off
; GCN-SDAG-REAL16-NEXT: s_set_pc_i64 s[30:31]
-;
-; GCN-GISEL-REAL16-LABEL: test_i8load_v4i8store:
-; GCN-GISEL-REAL16: ; %bb.0:
-; GCN-GISEL-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GCN-GISEL-REAL16-NEXT: s_wait_kmcnt 0x0
-; GCN-GISEL-REAL16-NEXT: global_load_u8 v6, v[4:5], off
-; GCN-GISEL-REAL16-NEXT: global_load_u8 v7, v[2:3], off
-; GCN-GISEL-REAL16-NEXT: global_load_u8 v10, v[0:1], off
-; GCN-GISEL-REAL16-NEXT: s_wait_loadcnt 0x2
-; GCN-GISEL-REAL16-NEXT: s_wait_xcnt 0x0
-; GCN-GISEL-REAL16-NEXT: v_lshlrev_b16 v0.l, 8, v6.l
-; GCN-GISEL-REAL16-NEXT: s_wait_loadcnt 0x0
-; GCN-GISEL-REAL16-NEXT: v_perm_b32 v1, v10, v7, 0xc0c0004
-; GCN-GISEL-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GCN-GISEL-REAL16-NEXT: v_or_b16 v0.l, v6.l, v0.l
-; GCN-GISEL-REAL16-NEXT: v_or_b16 v1.h, v0.l, v1.h
-; GCN-GISEL-REAL16-NEXT: global_store_b32 v[8:9], v1, off
-; GCN-GISEL-REAL16-NEXT: s_set_pc_i64 s[30:31]
%a = load i8, ptr addrspace(1) %ptr_a
%b = load i8, ptr addrspace(1) %ptr_b
%c = load i8, ptr addrspace(1) %ptr_c
diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
index 5aefa307f9117..643a95385f4e4 100644
--- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll
@@ -6,7 +6,7 @@
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX67,GFX7,GFX67-GISEL,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-amd-amdpal < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG,GFX900-SDAG,GFX900 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-amd-amdpal < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL,GFX900-GISEL,GFX900 %s
@@ -658,26 +658,48 @@ define <2 x i16> @clpeak_imad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) {
; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_imad_pat_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 1
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v0, v3
-; GFX8-NEXT: v_mad_u16 v0, v0, v3, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v2, v1
-; GFX8-NEXT: v_mad_u16 v2, v2, v1, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v2, v1
-; GFX8-NEXT: v_mad_u16 v2, v0, v4, v0
-; GFX8-NEXT: v_mad_u16 v3, v1, v5, v1
-; GFX8-NEXT: v_mad_u16 v0, v2, v0, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v1, v3, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: clpeak_imad_pat_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v0, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v3, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v2, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v2, v1, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v0, v4, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v1, v5, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v2, v0, v2
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v3, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_imad_pat_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v2, v1, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v0, v3, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v4, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v7, v5, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v2, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v4, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v5, v3, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v6, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v7, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v2, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: clpeak_imad_pat_v2i16:
; GFX9: ; %bb.0: ; %entry
@@ -762,91 +784,173 @@ entry:
}
define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
-; GFX67-LABEL: clpeak_imad_pat_v3i16:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_add_i32_e32 v4, vcc, 1, v4
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v4
-; GFX67-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v9, v7, v8
-; GFX67-NEXT: v_mad_u32_u24 v0, v5, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v4, v7, v8, v4
-; GFX67-NEXT: v_mad_u32_u24 v5, v5, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX67-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX67-NEXT: v_mad_u32_u24 v1, v6, v3, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v0, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v4, v4, v8
-; GFX67-NEXT: v_or_b32_e32 v5, v9, v5
-; GFX67-NEXT: s_mov_b32 s4, 0x10000
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_add_i32_e32 v5, vcc, s4, v5
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX67-NEXT: v_mul_u32_u24_e32 v8, v1, v3
-; GFX67-NEXT: v_mad_u32_u24 v6, v6, v3, 1
-; GFX67-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v1, v1, v3, 1
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v7
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v5
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v4, v2
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v8
-; GFX67-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v4, v4, v6
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v3, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v2, v5
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v3, v1
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_imad_pat_v3i16:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v4
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v9, v7, v8
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v5, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v7, v8, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v5, v5, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v6, v3, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v0, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v4, v4, v8
+; GFX67-SDAG-NEXT: v_or_b32_e32 v5, v9, v5
+; GFX67-SDAG-NEXT: s_mov_b32 s4, 0x10000
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v5, vcc, s4, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v8, v1, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v6, v6, v3, 1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v5
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v4, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v8
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v4, v4, v6
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v2, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_imad_pat_v3i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, 1
-; GFX8-NEXT: v_add_u16_e32 v4, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v6, v0, v5
-; GFX8-NEXT: v_mad_u16 v0, v0, v5, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v7, v1, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v8, v4, v2
-; GFX8-NEXT: v_mad_u16 v4, v4, v2, v4
-; GFX8-NEXT: v_mad_u16 v1, v1, v3, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v5
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v4, v2
-; GFX8-NEXT: v_mad_u16 v3, v0, v6, v0
-; GFX8-NEXT: v_mad_u16 v4, v2, v8, v2
-; GFX8-NEXT: v_mad_u16 v0, v3, v0, v3
-; GFX8-NEXT: v_mad_u16 v5, v1, v7, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v2, v4, v2, v4
-; GFX8-NEXT: v_mad_u16 v1, v5, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX67-GISEL-LABEL: clpeak_imad_pat_v3i16:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX67-GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX67-GISEL-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v6, v2, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v7, v5, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v8, v3, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v9, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v6, v6, v2, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v10, v4, v5
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v11, v1, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v7, v7, v5, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v8, v8, v3, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v7
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v11
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v8
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v5, v5, v6
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v3, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: clpeak_imad_pat_v3i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v5, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v4, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-SDAG-NEXT: v_add_u16_e32 v1, 1, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v6, v0, v5
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v5, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v7, v1, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v8, v4, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v4, v2, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v1, v3, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v0, v6, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v2, v8, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v3, v0, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v1, v7, v1
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v4, v2, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v5, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_imad_pat_v3i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v6, 1
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-GISEL-NEXT: v_add_u16_e32 v5, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_add_u16_e32 v1, 1, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v6, v5, v2, v5
+; GFX8-GISEL-NEXT: v_mad_u16 v7, v0, v4, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v8, v1, v3, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v9, v6, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v10, v7, v4
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v5, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v4, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v11, v8, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v1, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v6, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v7, v4, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v9, v5
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v10, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v8, v3, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v11, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: clpeak_imad_pat_v3i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1215,41 +1319,79 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX67-NEXT: v_or_b32_e32 v1, v1, v2
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_imad_pat_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, 1
-; GFX8-NEXT: v_add_u16_e32 v5, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v4, 1, v1
-; GFX8-NEXT: v_add_u16_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v9, v0, v8
-; GFX8-NEXT: v_mad_u16 v0, v0, v8, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v7, v1, v6
-; GFX8-NEXT: v_mul_lo_u16_e32 v10, v5, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v11, v4, v3
-; GFX8-NEXT: v_mad_u16 v1, v1, v6, v1
-; GFX8-NEXT: v_mad_u16 v4, v4, v3, v4
-; GFX8-NEXT: v_mad_u16 v5, v5, v2, v5
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v8
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v6
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v5, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v3, v4, v3
-; GFX8-NEXT: v_mad_u16 v4, v0, v9, v0
-; GFX8-NEXT: v_mad_u16 v5, v1, v7, v1
-; GFX8-NEXT: v_mad_u16 v7, v2, v10, v2
-; GFX8-NEXT: v_mad_u16 v0, v4, v0, v4
-; GFX8-NEXT: v_mad_u16 v6, v3, v11, v3
-; GFX8-NEXT: v_mad_u16 v1, v5, v1, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v2, v7, v2, v7
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_mad_u16 v2, v6, v3, v6
-; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: clpeak_imad_pat_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v6, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v5, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX8-SDAG-NEXT: v_add_u16_e32 v4, 1, v1
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v9, v0, v8
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v8, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v7, v1, v6
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v10, v5, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v11, v4, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v1, v6, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v4, v3, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v5, v2, v5
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v6
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v5, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v4, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v0, v9, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v1, v7, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v7, v2, v10, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v4, v0, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v6, v3, v11, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v5, v1, v5
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v7, v2, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v6, v3, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_imad_pat_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, 1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v4, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_add_u16_e32 v6, 1, v1
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v8, v4, v2, v4
+; GFX8-GISEL-NEXT: v_mad_u16 v9, v0, v5, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v10, v6, v3, v6
+; GFX8-GISEL-NEXT: v_mad_u16 v11, v1, v7, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v12, v8, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v13, v9, v5
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v4, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v5, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v14, v10, v3
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v15, v11, v7
+; GFX8-GISEL-NEXT: v_mad_u16 v6, v6, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v1, v7, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v8, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v9, v5, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v12, v4
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v13, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v10, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v7, v11, v7, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v14, v6
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v15, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v6, v3
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: clpeak_imad_pat_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -1883,26 +2025,48 @@ define <2 x i16> @clpeak_umad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) {
; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_umad_pat_v2i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 1
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v0, v3
-; GFX8-NEXT: v_mad_u16 v0, v0, v3, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v2, v1
-; GFX8-NEXT: v_mad_u16 v2, v2, v1, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v2, v1
-; GFX8-NEXT: v_mad_u16 v2, v0, v4, v0
-; GFX8-NEXT: v_mad_u16 v3, v1, v5, v1
-; GFX8-NEXT: v_mad_u16 v0, v2, v0, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v1, v3, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: clpeak_umad_pat_v2i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v0, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v3, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v2, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v2, v1, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v0, v4, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v1, v5, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v2, v0, v2
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v3, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_umad_pat_v2i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v2, v1, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v0, v3, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v4, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v7, v5, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v2, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v4, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v5, v3, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v6, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v7, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v2, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: clpeak_umad_pat_v2i16:
; GFX9: ; %bb.0: ; %entry
@@ -1987,91 +2151,173 @@ entry:
}
define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) {
-; GFX67-LABEL: clpeak_umad_pat_v3i16:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX67-NEXT: v_add_i32_e32 v4, vcc, 1, v4
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v7, 0xffff, v4
-; GFX67-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v9, v7, v8
-; GFX67-NEXT: v_mad_u32_u24 v0, v5, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v4, v7, v8, v4
-; GFX67-NEXT: v_mad_u32_u24 v5, v5, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX67-NEXT: v_lshlrev_b32_e32 v9, 16, v9
-; GFX67-NEXT: v_mad_u32_u24 v1, v6, v3, v1
-; GFX67-NEXT: v_mul_u32_u24_e32 v7, v0, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v4, v4, v8
-; GFX67-NEXT: v_or_b32_e32 v5, v9, v5
-; GFX67-NEXT: s_mov_b32 s4, 0x10000
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v2, 1
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_add_i32_e32 v5, vcc, s4, v5
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX67-NEXT: v_mul_u32_u24_e32 v8, v1, v3
-; GFX67-NEXT: v_mad_u32_u24 v6, v6, v3, 1
-; GFX67-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX67-NEXT: v_mad_u32_u24 v1, v1, v3, 1
-; GFX67-NEXT: v_lshrrev_b32_e32 v2, 16, v5
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v7
-; GFX67-NEXT: v_and_b32_e32 v5, 0xffff, v5
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v3, v3, v5
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v4, v2
-; GFX67-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX67-NEXT: v_and_b32_e32 v4, 0xffff, v8
-; GFX67-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX67-NEXT: v_mul_u32_u24_e32 v4, v4, v6
-; GFX67-NEXT: v_mul_u32_u24_e32 v0, v3, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v2, v2, v5
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_and_b32_e32 v3, 0xffff, v4
-; GFX67-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_mul_u32_u24_e32 v1, v3, v1
-; GFX67-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: clpeak_umad_pat_v3i16:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-SDAG-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v7, 0xffff, v4
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v9, v7, v8
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v5, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v4, v7, v8, v4
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v5, v5, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v9, 16, v9
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v6, v3, v1
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v7, v0, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v4, v4, v8
+; GFX67-SDAG-NEXT: v_or_b32_e32 v5, v9, v5
+; GFX67-SDAG-NEXT: s_mov_b32 s4, 0x10000
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_add_i32_e32 v5, vcc, s4, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v8, v1, v3
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v6, v6, v3, 1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v7
+; GFX67-SDAG-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-SDAG-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v3, v3, v5
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v4, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX67-SDAG-NEXT: v_and_b32_e32 v4, 0xffff, v8
+; GFX67-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v4, v4, v6
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v2, v2, v5
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_umad_pat_v3i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v5, 1
-; GFX8-NEXT: v_add_u16_e32 v4, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v1, 1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v6, v0, v5
-; GFX8-NEXT: v_mad_u16 v0, v0, v5, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v7, v1, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v8, v4, v2
-; GFX8-NEXT: v_mad_u16 v4, v4, v2, v4
-; GFX8-NEXT: v_mad_u16 v1, v1, v3, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v5
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v4, v2
-; GFX8-NEXT: v_mad_u16 v3, v0, v6, v0
-; GFX8-NEXT: v_mad_u16 v4, v2, v8, v2
-; GFX8-NEXT: v_mad_u16 v0, v3, v0, v3
-; GFX8-NEXT: v_mad_u16 v5, v1, v7, v1
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v2, v4, v2, v4
-; GFX8-NEXT: v_mad_u16 v1, v5, v1, v5
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX67-GISEL-LABEL: clpeak_umad_pat_v3i16:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX67-GISEL-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX67-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX67-GISEL-NEXT: v_add_i32_e32 v4, vcc, 1, v4
+; GFX67-GISEL-NEXT: v_add_i32_e32 v1, vcc, 1, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v7, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v8, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v6, v2, v0
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v4, v7, v5, v4
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v8, v3, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v9, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v6, v6, v2, 1
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v10, v4, v5
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v11, v1, v3
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v7, v7, v5, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v8, v8, v3, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v2, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v4, v5, 1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v1, v1, v3, 1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v9
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v3, v3, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v10
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v7
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v4, v4, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v5, 0xffff, v11
+; GFX67-GISEL-NEXT: v_and_b32_e32 v6, 0xffff, v8
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v0, v3, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v4
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v5, v5, v6
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v2, v3, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v5
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX67-GISEL-NEXT: v_mul_u32_u24_e32 v1, v3, v1
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: clpeak_umad_pat_v3i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v5, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v4, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-SDAG-NEXT: v_add_u16_e32 v1, 1, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v6, v0, v5
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v5, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v7, v1, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v8, v4, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v4, v2, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v1, v3, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v0, v6, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v2, v8, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v3, v0, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v1, v7, v1
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v4, v2, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v5, v1, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_umad_pat_v3i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v6, 1
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v2
+; GFX8-GISEL-NEXT: v_add_u16_e32 v5, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_add_u16_e32 v1, 1, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v6, v5, v2, v5
+; GFX8-GISEL-NEXT: v_mad_u16 v7, v0, v4, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v8, v1, v3, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v9, v6, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v10, v7, v4
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v5, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v4, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v11, v8, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v1, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v6, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v7, v4, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v9, v5
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v10, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v8, v3, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v11, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v5, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: clpeak_umad_pat_v3i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -2440,41 +2686,79 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) {
; GFX67-NEXT: v_or_b32_e32 v1, v1, v2
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_umad_pat_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v6, 1
-; GFX8-NEXT: v_add_u16_e32 v5, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GFX8-NEXT: v_add_u16_e32 v4, 1, v1
-; GFX8-NEXT: v_add_u16_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT: v_mul_lo_u16_e32 v9, v0, v8
-; GFX8-NEXT: v_mad_u16 v0, v0, v8, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v7, v1, v6
-; GFX8-NEXT: v_mul_lo_u16_e32 v10, v5, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v11, v4, v3
-; GFX8-NEXT: v_mad_u16 v1, v1, v6, v1
-; GFX8-NEXT: v_mad_u16 v4, v4, v3, v4
-; GFX8-NEXT: v_mad_u16 v5, v5, v2, v5
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v8
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v6
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v5, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v3, v4, v3
-; GFX8-NEXT: v_mad_u16 v4, v0, v9, v0
-; GFX8-NEXT: v_mad_u16 v5, v1, v7, v1
-; GFX8-NEXT: v_mad_u16 v7, v2, v10, v2
-; GFX8-NEXT: v_mad_u16 v0, v4, v0, v4
-; GFX8-NEXT: v_mad_u16 v6, v3, v11, v3
-; GFX8-NEXT: v_mad_u16 v1, v5, v1, v5
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v2, v7, v2, v7
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT: v_mad_u16 v2, v6, v3, v6
-; GFX8-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: clpeak_umad_pat_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v6, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v5, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GFX8-SDAG-NEXT: v_add_u16_e32 v4, 1, v1
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v1, v1, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v9, v0, v8
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v8, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v7, v1, v6
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v10, v5, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v11, v4, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v1, v6, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v4, v3, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v5, v2, v5
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v8
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v6
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v5, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v4, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v0, v9, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v1, v7, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v7, v2, v10, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v4, v0, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v6, v3, v11, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v5, v1, v5
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v7, v2, v7
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v6, v3, v6
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_umad_pat_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, 1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v4, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_add_u16_e32 v6, 1, v1
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v2
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v8, v4, v2, v4
+; GFX8-GISEL-NEXT: v_mad_u16 v9, v0, v5, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v10, v6, v3, v6
+; GFX8-GISEL-NEXT: v_mad_u16 v11, v1, v7, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v12, v8, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v13, v9, v5
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v4, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v5, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v14, v10, v3
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v15, v11, v7
+; GFX8-GISEL-NEXT: v_mad_u16 v6, v6, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v1, v7, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v8, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v9, v5, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v12, v4
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v13, v0
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v10, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v7, v11, v7, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v14, v6
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v15, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v6, v3
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v1, v1, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: clpeak_umad_pat_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -10286,38 +10570,72 @@ define <2 x i16> @clpeak_imad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) {
; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_imad_pat_v2i16_x2:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 1
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_mad_u16 v4, v0, v3, v0
-; GFX8-NEXT: v_mad_u16 v5, v2, v1, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v4, v3
-; GFX8-NEXT: v_mad_u16 v0, v0, v3, 1
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v5, v1
-; GFX8-NEXT: v_mad_u16 v1, v2, v1, 1
-; GFX8-NEXT: v_mad_u16 v2, v4, v0, v0
-; GFX8-NEXT: v_mad_u16 v3, v5, v1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v2, v4
-; GFX8-NEXT: v_mad_u16 v0, v4, v0, 1
-; GFX8-NEXT: v_mul_lo_u16_e32 v3, v3, v5
-; GFX8-NEXT: v_mad_u16 v1, v5, v1, 1
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v2, v0
-; GFX8-NEXT: v_mad_u16 v0, v2, v0, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v3, v1
-; GFX8-NEXT: v_mad_u16 v1, v3, v1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_mad_u16 v2, v0, v4, v0
-; GFX8-NEXT: v_mad_u16 v3, v1, v5, v1
-; GFX8-NEXT: v_mad_u16 v0, v2, v0, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v1, v3, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: clpeak_imad_pat_v2i16_x2:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v0, v3, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v2, v1, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v4, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v3, 1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v5, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v2, v1, 1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v4, v0, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v5, v1, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v4, v0, 1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v5
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v5, v1, 1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v2, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v2, v0, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v3, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v3, v1, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v0, v4, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v1, v5, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v2, v0, v2
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v3, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_imad_pat_v2i16_x2:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v2, v1, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v0, v3, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v4, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v5, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v2, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v4, v1, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v5, v0, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v4
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v5
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v4, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v5, v0, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v2, v1, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v3, v0, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v4, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v7, v5, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v2, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v3, v0, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v4, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v5, v3, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v6, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v7, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: clpeak_imad_pat_v2i16_x2:
; GFX9: ; %bb.0: ; %entry
@@ -10507,38 +10825,72 @@ define <2 x i16> @clpeak_umad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) {
; GFX67-NEXT: v_or_b32_e32 v0, v0, v1
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: clpeak_umad_pat_v2i16_x2:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, 1
-; GFX8-NEXT: v_add_u16_e32 v2, 1, v0
-; GFX8-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX8-NEXT: v_mad_u16 v4, v0, v3, v0
-; GFX8-NEXT: v_mad_u16 v5, v2, v1, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v4, v3
-; GFX8-NEXT: v_mad_u16 v0, v0, v3, 1
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v5, v1
-; GFX8-NEXT: v_mad_u16 v1, v2, v1, 1
-; GFX8-NEXT: v_mad_u16 v2, v4, v0, v0
-; GFX8-NEXT: v_mad_u16 v3, v5, v1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v2, v2, v4
-; GFX8-NEXT: v_mad_u16 v0, v4, v0, 1
-; GFX8-NEXT: v_mul_lo_u16_e32 v3, v3, v5
-; GFX8-NEXT: v_mad_u16 v1, v5, v1, 1
-; GFX8-NEXT: v_mul_lo_u16_e32 v4, v2, v0
-; GFX8-NEXT: v_mad_u16 v0, v2, v0, v0
-; GFX8-NEXT: v_mul_lo_u16_e32 v5, v3, v1
-; GFX8-NEXT: v_mad_u16 v1, v3, v1, v1
-; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v2
-; GFX8-NEXT: v_mul_lo_u16_e32 v1, v1, v3
-; GFX8-NEXT: v_mad_u16 v2, v0, v4, v0
-; GFX8-NEXT: v_mad_u16 v3, v1, v5, v1
-; GFX8-NEXT: v_mad_u16 v0, v2, v0, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX8-NEXT: v_mad_u16 v1, v3, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: clpeak_umad_pat_v2i16_x2:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-SDAG-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-SDAG-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v0, v3, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v5, v2, v1, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v4, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v3, 1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v5, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v2, v1, 1
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v4, v0, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v5, v1, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v4, v0, 1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v5
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v5, v1, 1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v2, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v2, v0, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v3, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v3, v1, v1
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v0, v4, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v1, v5, v1
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v2, v0, v2
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-SDAG-NEXT: v_mad_u16 v1, v3, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: clpeak_umad_pat_v2i16_x2:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 1
+; GFX8-GISEL-NEXT: v_add_u16_e32 v2, 1, v0
+; GFX8-GISEL-NEXT: v_add_u16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v2, v1, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v0, v3, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v4, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v5, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v2, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v3, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v4, v1, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v5, v0, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v4
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v5
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v4, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v5, v0, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v4, v2, v1, v1
+; GFX8-GISEL-NEXT: v_mad_u16 v5, v3, v0, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v4, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v7, v5, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v2, v1, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v3, v0, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v4, v2, 1
+; GFX8-GISEL-NEXT: v_mad_u16 v3, v5, v3, 1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v6, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v7, v0
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v2
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: clpeak_umad_pat_v2i16_x2:
; GFX9: ; %bb.0: ; %entry
@@ -10740,26 +11092,48 @@ entry:
}
define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) {
-; GFX67-LABEL: multi_use_mul_mad_i16_var:
-; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX67-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX67-NEXT: v_mad_u32_u24 v2, v0, v1, v2
-; GFX67-NEXT: v_mad_u32_u24 v0, v0, v1, v3
-; GFX67-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX67-NEXT: v_and_b32_e32 v1, 0xffff, v2
-; GFX67-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX67-NEXT: s_setpc_b64 s[30:31]
+; GFX67-SDAG-LABEL: multi_use_mul_mad_i16_var:
+; GFX67-SDAG: ; %bb.0: ; %entry
+; GFX67-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v2, v0, v1, v2
+; GFX67-SDAG-NEXT: v_mad_u32_u24 v0, v0, v1, v3
+; GFX67-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX67-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX67-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX67-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: multi_use_mul_mad_i16_var:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mad_u16 v3, v0, v1, v3
-; GFX8-NEXT: v_mad_u16 v0, v0, v1, v2
-; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX67-GISEL-LABEL: multi_use_mul_mad_i16_var:
+; GFX67-GISEL: ; %bb.0: ; %entry
+; GFX67-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v2, v0, v1, v2
+; GFX67-GISEL-NEXT: v_mad_u32_u24 v0, v0, v1, v3
+; GFX67-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX67-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX67-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX67-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX67-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: multi_use_mul_mad_i16_var:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mad_u16 v3, v0, v1, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v1, v2
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: multi_use_mul_mad_i16_var:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v0, v1, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v1, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: multi_use_mul_mad_i16_var:
; GFX9-SDAG: ; %bb.0: ; %entry
@@ -11201,23 +11575,41 @@ define <4 x i16> @multi_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i
; GFX67-NEXT: v_or_b32_e32 v1, v1, v2
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: multi_use_mul_mad_v2i16_var:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v2
-; GFX8-NEXT: v_mad_u16 v6, v5, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v6, 16, v6
-; GFX8-NEXT: v_mad_u16 v2, v0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX8-NEXT: v_lshrrev_b32_e32 v6, 16, v3
-; GFX8-NEXT: v_mad_u16 v4, v5, v4, v6
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_mad_u16 v0, v0, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v1, v0, v4
-; GFX8-NEXT: v_mov_b32_e32 v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: multi_use_mul_mad_v2i16_var:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v6, v5, v4, v6
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX8-SDAG-NEXT: v_mad_u16 v2, v0, v1, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v5, v4, v6
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v0, v4
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: multi_use_mul_mad_v2i16_var:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v6, v4, v5, v6
+; GFX8-GISEL-NEXT: v_mad_u16 v2, v0, v1, v2
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; GFX8-GISEL-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v1, v3
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v4, v5, v6
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v1, v0, v1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, v2
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: multi_use_mul_mad_v2i16_var:
; GFX9: ; %bb.0: ; %entry
@@ -11298,23 +11690,41 @@ define <2 x i16> @other_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
; GFX67-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: other_use_mul_mad_v2i16_var:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX8-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX8-NEXT: v_mul_lo_u16_sdwa v6, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mul_lo_u16_e32 v7, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v6, v7, v6
-; GFX8-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX8-NEXT: v_mad_u16 v4, v5, v4, v7
-; GFX8-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX8-NEXT: v_mad_u16 v0, v0, v1, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: ds_write_b32 v3, v6
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: other_use_mul_mad_v2i16_var:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v1
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v6, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v7, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v6, v7, v6
+; GFX8-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX8-SDAG-NEXT: v_mad_u16 v4, v5, v4, v7
+; GFX8-SDAG-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX8-SDAG-NEXT: v_mad_u16 v0, v0, v1, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: s_mov_b32 m0, -1
+; GFX8-SDAG-NEXT: ds_write_b32 v3, v6
+; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: other_use_mul_mad_v2i16_var:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v0, v1
+; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v7, v4, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v6, v6, v7
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v0, v0, v1, v2
+; GFX8-GISEL-NEXT: v_mad_u16 v1, v4, v5, v7
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: s_mov_b32 m0, -1
+; GFX8-GISEL-NEXT: ds_write_b32 v3, v6
+; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: other_use_mul_mad_v2i16_var:
; GFX9: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 2fef3bc734776..d8117b76fb576 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-SDAG,GFX1013-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-GISEL,GFX1013-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.13 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-GISEL,GFX1013-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-SDAG,GFX1030-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-GISEL,GFX1030-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.30 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX10,GFX10-GISEL,GFX1030-GISEL %s
; RUN: not llc -mtriple=amdgpu10.12 < %s 2>&1 | FileCheck -check-prefix=ERR %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=PRE-GFX12,GFX11-GISEL %s
@@ -203,26 +203,98 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_ssssss(i32 inreg %node_ptr
}
define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 inreg %node_ptr, float inreg %ray_extent, <3 x float> inreg %ray_origin, <3 x half> inreg %ray_dir, <3 x half> inreg %ray_inv_dir, <4 x i32> inreg %tdescr) {
-; GFX10-LABEL: image_bvh_intersect_ray_a16:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_mov_b32 s15, s12
-; GFX10-NEXT: s_mov_b32 s12, s9
-; GFX10-NEXT: s_lshr_b32 s9, s7, 16
-; GFX10-NEXT: s_pack_ll_b32_b16 s6, s6, s7
-; GFX10-NEXT: s_pack_ll_b32_b16 s7, s9, s8
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: s_mov_b32 s14, s11
-; GFX10-NEXT: s_mov_b32 s13, s10
-; GFX10-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[12:15] a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
+; GFX10-SDAG-LABEL: image_bvh_intersect_ray_a16:
+; GFX10-SDAG: ; %bb.0: ; %main_body
+; GFX10-SDAG-NEXT: s_mov_b32 s15, s12
+; GFX10-SDAG-NEXT: s_mov_b32 s12, s9
+; GFX10-SDAG-NEXT: s_lshr_b32 s9, s7, 16
+; GFX10-SDAG-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX10-SDAG-NEXT: s_pack_ll_b32_b16 s7, s9, s8
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v5, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v7, s7
+; GFX10-SDAG-NEXT: s_mov_b32 s14, s11
+; GFX10-SDAG-NEXT: s_mov_b32 s13, s10
+; GFX10-SDAG-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[12:15] a16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1013-GISEL-LABEL: image_bvh_intersect_ray_a16:
+; GFX1013-GISEL: ; %bb.0: ; %main_body
+; GFX1013-GISEL-NEXT: s_and_b32 s8, s8, 0xffff
+; GFX1013-GISEL-NEXT: s_mov_b32 s16, s9
+; GFX1013-GISEL-NEXT: v_alignbit_b32 v0, s8, s7, 16
+; GFX1013-GISEL-NEXT: s_lshr_b32 s9, s5, 16
+; GFX1013-GISEL-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX1013-GISEL-NEXT: s_lshl_b32 s8, s9, 16
+; GFX1013-GISEL-NEXT: s_and_b32 s9, s7, 0xffff
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s7, v0
+; GFX1013-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1013-GISEL-NEXT: s_lshl_b32 s9, s9, 16
+; GFX1013-GISEL-NEXT: s_or_b32 s5, s5, s8
+; GFX1013-GISEL-NEXT: s_or_b32 s6, s6, s9
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, s7
+; GFX1013-GISEL-NEXT: s_mov_b32 s17, s10
+; GFX1013-GISEL-NEXT: s_mov_b32 s18, s11
+; GFX1013-GISEL-NEXT: s_mov_b32 s19, s12
+; GFX1013-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
+; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1013-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1030-GISEL-LABEL: image_bvh_intersect_ray_a16:
+; GFX1030-GISEL: ; %bb.0: ; %main_body
+; GFX1030-GISEL-NEXT: s_mov_b32 s16, s9
+; GFX1030-GISEL-NEXT: s_lshr_b32 s9, s5, 16
+; GFX1030-GISEL-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX1030-GISEL-NEXT: s_lshl_b32 s9, s9, 16
+; GFX1030-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1030-GISEL-NEXT: s_or_b32 s5, s5, s9
+; GFX1030-GISEL-NEXT: s_and_b32 s9, s7, 0xffff
+; GFX1030-GISEL-NEXT: s_and_b32 s8, s8, 0xffff
+; GFX1030-GISEL-NEXT: s_lshl_b32 s9, s9, 16
+; GFX1030-GISEL-NEXT: v_alignbit_b32 v7, s8, s7, 16
+; GFX1030-GISEL-NEXT: s_or_b32 s6, s6, s9
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX1030-GISEL-NEXT: s_mov_b32 s17, s10
+; GFX1030-GISEL-NEXT: s_mov_b32 s18, s11
+; GFX1030-GISEL-NEXT: s_mov_b32 s19, s12
+; GFX1030-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[16:19] a16
+; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1030-GISEL-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: image_bvh_intersect_ray_a16:
; GFX11-SDAG: ; %bb.0: ; %main_body
@@ -432,27 +504,101 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_ssssss(i64 inreg %node_p
}
define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 inreg %node_ptr, float inreg %ray_extent, <3 x float> inreg %ray_origin, <3 x half> inreg %ray_dir, <3 x half> inreg %ray_inv_dir, <4 x i32> inreg %tdescr) {
-; GFX10-LABEL: image_bvh64_intersect_ray_a16:
-; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_mov_b32 s14, s12
-; GFX10-NEXT: s_mov_b32 s12, s10
-; GFX10-NEXT: s_lshr_b32 s10, s8, 16
-; GFX10-NEXT: s_pack_ll_b32_b16 s7, s7, s8
-; GFX10-NEXT: s_pack_ll_b32_b16 s8, s10, s9
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: v_mov_b32_e32 v8, s8
-; GFX10-NEXT: s_mov_b32 s15, s13
-; GFX10-NEXT: s_mov_b32 s13, s11
-; GFX10-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[12:15] a16
-; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: ; return to shader part epilog
+; GFX10-SDAG-LABEL: image_bvh64_intersect_ray_a16:
+; GFX10-SDAG: ; %bb.0: ; %main_body
+; GFX10-SDAG-NEXT: s_mov_b32 s14, s12
+; GFX10-SDAG-NEXT: s_mov_b32 s12, s10
+; GFX10-SDAG-NEXT: s_lshr_b32 s10, s8, 16
+; GFX10-SDAG-NEXT: s_pack_ll_b32_b16 s7, s7, s8
+; GFX10-SDAG-NEXT: s_pack_ll_b32_b16 s8, s10, s9
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v5, s5
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v7, s7
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v8, s8
+; GFX10-SDAG-NEXT: s_mov_b32 s15, s13
+; GFX10-SDAG-NEXT: s_mov_b32 s13, s11
+; GFX10-SDAG-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[12:15] a16
+; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX10-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX1013-GISEL-LABEL: image_bvh64_intersect_ray_a16:
+; GFX1013-GISEL: ; %bb.0: ; %main_body
+; GFX1013-GISEL-NEXT: s_and_b32 s9, s9, 0xffff
+; GFX1013-GISEL-NEXT: s_mov_b32 s16, s10
+; GFX1013-GISEL-NEXT: v_alignbit_b32 v0, s9, s8, 16
+; GFX1013-GISEL-NEXT: s_lshr_b32 s10, s6, 16
+; GFX1013-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1013-GISEL-NEXT: s_lshl_b32 s9, s10, 16
+; GFX1013-GISEL-NEXT: s_and_b32 s10, s8, 0xffff
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s8, v0
+; GFX1013-GISEL-NEXT: s_and_b32 s7, s7, 0xffff
+; GFX1013-GISEL-NEXT: s_lshl_b32 s10, s10, 16
+; GFX1013-GISEL-NEXT: s_or_b32 s6, s6, s9
+; GFX1013-GISEL-NEXT: s_or_b32 s7, s7, s10
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, s7
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v8, s8
+; GFX1013-GISEL-NEXT: s_mov_b32 s17, s11
+; GFX1013-GISEL-NEXT: s_mov_b32 s18, s12
+; GFX1013-GISEL-NEXT: s_mov_b32 s19, s13
+; GFX1013-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
+; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1013-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1013-GISEL-NEXT: ; return to shader part epilog
+;
+; GFX1030-GISEL-LABEL: image_bvh64_intersect_ray_a16:
+; GFX1030-GISEL: ; %bb.0: ; %main_body
+; GFX1030-GISEL-NEXT: s_mov_b32 s16, s10
+; GFX1030-GISEL-NEXT: s_lshr_b32 s10, s6, 16
+; GFX1030-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX1030-GISEL-NEXT: s_lshl_b32 s10, s10, 16
+; GFX1030-GISEL-NEXT: s_and_b32 s7, s7, 0xffff
+; GFX1030-GISEL-NEXT: s_or_b32 s6, s6, s10
+; GFX1030-GISEL-NEXT: s_and_b32 s10, s8, 0xffff
+; GFX1030-GISEL-NEXT: s_and_b32 s9, s9, 0xffff
+; GFX1030-GISEL-NEXT: s_lshl_b32 s10, s10, 16
+; GFX1030-GISEL-NEXT: v_alignbit_b32 v8, s9, s8, 16
+; GFX1030-GISEL-NEXT: s_or_b32 s7, s7, s10
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, s4
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, s5
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, s6
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, s7
+; GFX1030-GISEL-NEXT: s_mov_b32 s17, s11
+; GFX1030-GISEL-NEXT: s_mov_b32 s18, s12
+; GFX1030-GISEL-NEXT: s_mov_b32 s19, s13
+; GFX1030-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[16:19] a16
+; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; GFX1030-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1030-GISEL-NEXT: ; return to shader part epilog
;
; GFX11-SDAG-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-SDAG: ; %bb.0: ; %main_body
@@ -794,20 +940,24 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1013-GISEL-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX1013-GISEL: ; %bb.0: ; %main_body
; GFX1013-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v6, 2, v0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, 0x48004700
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, 0x46004500
; GFX1013-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-GISEL-NEXT: v_add_co_u32 v2, s0, s8, v0
-; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e64 v3, s0, s9, 0, s0
-; GFX1013-GISEL-NEXT: v_add_co_u32 v4, s0, s10, v0
-; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e64 v5, s0, s11, 0, s0
-; GFX1013-GISEL-NEXT: flat_load_dword v0, v[2:3]
-; GFX1013-GISEL-NEXT: flat_load_dword v1, v[4:5]
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s8
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s9
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, s10
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, s11
+; GFX1013-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, v6
+; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo
+; GFX1013-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v6
+; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, 0x46004500
+; GFX1013-GISEL-NEXT: flat_load_dword v0, v[4:5]
+; GFX1013-GISEL-NEXT: flat_load_dword v1, v[2:3]
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, 0x44004200
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, 1.0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, 2.0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, 0x44004200
; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1013-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[12:15] a16
; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -840,20 +990,24 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
; GFX1030-GISEL-LABEL: image_bvh_intersect_ray_a16_nsa_reassign:
; GFX1030-GISEL: ; %bb.0: ; %main_body
; GFX1030-GISEL-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
-; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, 0x48004700
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, 0x46004500
+; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v4, 2, v0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, 0x44004200
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, 2.0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, 0x46004500
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, 0x48004700
; GFX1030-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-GISEL-NEXT: v_add_co_u32 v0, s0, s0, v2
-; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s1, 0, s0
-; GFX1030-GISEL-NEXT: v_add_co_u32 v2, s0, s2, v2
-; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, s3, 0, s0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, s3
+; GFX1030-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
+; GFX1030-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, vcc_lo
; GFX1030-GISEL-NEXT: flat_load_dword v0, v[0:1]
; GFX1030-GISEL-NEXT: flat_load_dword v1, v[2:3]
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, 1.0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, 2.0
; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1030-GISEL-NEXT: image_bvh_intersect_ray v[0:3], v[0:7], s[4:7] a16
; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1213,19 +1367,21 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1013-GISEL-NEXT: s_clause 0x1
; GFX1013-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1013-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1013-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v8, 0x48004700
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, 0x46004500
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, 0x44004200
-; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v6, 0x44004200
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v7, 0x46004500
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1013-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1013-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e64 v1, s4, s7, 0, s4
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX1013-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX1013-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1013-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
; GFX1013-GISEL-NEXT: flat_load_dword v2, v[0:1]
-; GFX1013-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1013-GISEL-NEXT: v_mov_b32_e32 v0, 0xb36211c6
+; GFX1013-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1013-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[0:3] a16
; GFX1013-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1261,19 +1417,21 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
; GFX1030-GISEL-NEXT: s_clause 0x1
; GFX1030-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX1030-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; GFX1030-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v3, 0
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v8, 0x48004700
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, 0x46004500
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, 0x44004200
-; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v4, 1.0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v5, 2.0
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v6, 0x44004200
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v7, 0x46004500
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v8, 0x48004700
; GFX1030-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-GISEL-NEXT: v_add_co_u32 v0, s4, s6, v0
-; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, s7, 0, s4
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, s6
+; GFX1030-GISEL-NEXT: v_mov_b32_e32 v1, s7
+; GFX1030-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2
+; GFX1030-GISEL-NEXT: v_add_co_ci_u32_e64 v1, null, 0, v1, vcc_lo
; GFX1030-GISEL-NEXT: flat_load_dword v2, v[0:1]
-; GFX1030-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1030-GISEL-NEXT: v_mov_b32_e32 v0, 0xb36211c6
+; GFX1030-GISEL-NEXT: v_bfrev_b32_e32 v1, 4.0
; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX1030-GISEL-NEXT: image_bvh64_intersect_ray v[0:3], v[0:8], s[0:3] a16
; GFX1030-GISEL-NEXT: s_waitcnt vmcnt(0)
@@ -1350,3 +1508,5 @@ main_body:
}
declare i32 @llvm.amdgcn.workitem.id.x()
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX10: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
index bb9c991cd6e65..e56b7dd9f7f6e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -1745,22 +1745,134 @@ define void @test_readfirstlane_v32f16(ptr addrspace(1) %out, <32 x half> %src)
; CHECK-GISEL-LABEL: test_readfirstlane_v32f16:
; CHECK-GISEL: ; %bb.0:
; CHECK-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s19, v17
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s18, v16
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s17, v15
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s16, v14
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s15, v13
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s14, v12
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s13, v11
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s12, v10
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s11, v9
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s10, v8
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s9, v7
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s8, v6
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s7, v5
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v2
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v18, 16, v4
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v19, 16, v5
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s4, v0
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v4, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s5, v1
+; CHECK-GISEL-NEXT: s_lshr_b32 s20, s4, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v5, v5, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; CHECK-GISEL-NEXT: v_readfirstlane_b32 s6, v4
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s5, v3
-; CHECK-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; CHECK-GISEL-NEXT: s_lshr_b32 s21, s5, 16
+; CHECK-GISEL-NEXT: s_and_b32 s4, s4, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s20, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v18, 16, v8
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v2, v6, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s7, v5
+; CHECK-GISEL-NEXT: s_lshr_b32 s22, s6, 16
+; CHECK-GISEL-NEXT: s_or_b32 s4, s4, s20
+; CHECK-GISEL-NEXT: s_and_b32 s5, s5, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s21, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v19, 16, v9
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v3, v7, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s8, v2
+; CHECK-GISEL-NEXT: s_lshr_b32 s23, s7, 16
+; CHECK-GISEL-NEXT: s_or_b32 s5, s5, s20
+; CHECK-GISEL-NEXT: s_and_b32 s6, s6, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s22, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v6, 16, v10
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v8, v8, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s9, v3
+; CHECK-GISEL-NEXT: s_lshr_b32 s24, s8, 16
+; CHECK-GISEL-NEXT: s_or_b32 s6, s6, s20
+; CHECK-GISEL-NEXT: s_and_b32 s7, s7, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s23, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v11
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v9, v9, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v6
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s10, v8
+; CHECK-GISEL-NEXT: s_lshr_b32 s25, s9, 16
+; CHECK-GISEL-NEXT: s_or_b32 s7, s7, s20
+; CHECK-GISEL-NEXT: s_and_b32 s8, s8, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s24, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v18, 16, v12
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v6, v10, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s11, v9
+; CHECK-GISEL-NEXT: s_lshr_b32 s26, s10, 16
+; CHECK-GISEL-NEXT: s_or_b32 s8, s8, s20
+; CHECK-GISEL-NEXT: s_and_b32 s9, s9, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s25, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v19, 16, v13
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v7, v11, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v18, 16, v18
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s12, v6
+; CHECK-GISEL-NEXT: s_lshr_b32 s27, s11, 16
+; CHECK-GISEL-NEXT: s_or_b32 s9, s9, s20
+; CHECK-GISEL-NEXT: s_and_b32 s10, s10, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s26, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v10, 16, v14
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v12, v12, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v19, 16, v19
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s13, v7
+; CHECK-GISEL-NEXT: s_lshr_b32 s28, s12, 16
+; CHECK-GISEL-NEXT: s_or_b32 s10, s10, s20
+; CHECK-GISEL-NEXT: s_and_b32 s11, s11, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s27, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v11, 16, v15
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v13, v13, v19 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v10
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s14, v12
+; CHECK-GISEL-NEXT: s_lshr_b32 s29, s13, 16
+; CHECK-GISEL-NEXT: s_or_b32 s11, s11, s20
+; CHECK-GISEL-NEXT: s_and_b32 s12, s12, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s28, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v18, 16, v16
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v10, v14, v10 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v11, 16, v11
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s15, v13
+; CHECK-GISEL-NEXT: s_lshr_b32 s40, s14, 16
+; CHECK-GISEL-NEXT: s_or_b32 s12, s12, s20
+; CHECK-GISEL-NEXT: s_and_b32 s13, s13, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s29, 16
+; CHECK-GISEL-NEXT: v_lshrrev_b32_e32 v19, 16, v17
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v11, v15, v11 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v14, 16, v18
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s16, v10
+; CHECK-GISEL-NEXT: s_lshr_b32 s41, s15, 16
+; CHECK-GISEL-NEXT: s_or_b32 s13, s13, s20
+; CHECK-GISEL-NEXT: s_and_b32 s14, s14, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s40, 16
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v14, v16, v14 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_lshlrev_b32_e32 v15, 16, v19
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s17, v11
+; CHECK-GISEL-NEXT: s_lshr_b32 s42, s16, 16
+; CHECK-GISEL-NEXT: s_or_b32 s14, s14, s20
+; CHECK-GISEL-NEXT: s_and_b32 s15, s15, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s41, 16
+; CHECK-GISEL-NEXT: v_or_b32_sdwa v15, v17, v15 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s18, v14
+; CHECK-GISEL-NEXT: s_lshr_b32 s43, s17, 16
+; CHECK-GISEL-NEXT: s_or_b32 s15, s15, s20
+; CHECK-GISEL-NEXT: s_and_b32 s16, s16, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s42, 16
+; CHECK-GISEL-NEXT: v_readfirstlane_b32 s19, v15
+; CHECK-GISEL-NEXT: s_lshr_b32 s44, s18, 16
+; CHECK-GISEL-NEXT: s_or_b32 s16, s16, s20
+; CHECK-GISEL-NEXT: s_and_b32 s17, s17, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s43, 16
+; CHECK-GISEL-NEXT: s_lshr_b32 s45, s19, 16
+; CHECK-GISEL-NEXT: s_or_b32 s17, s17, s20
+; CHECK-GISEL-NEXT: s_and_b32 s18, s18, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s44, 16
+; CHECK-GISEL-NEXT: s_or_b32 s18, s18, s20
+; CHECK-GISEL-NEXT: s_and_b32 s19, s19, 0xffff
+; CHECK-GISEL-NEXT: s_lshl_b32 s20, s45, 16
+; CHECK-GISEL-NEXT: s_or_b32 s19, s19, s20
; CHECK-GISEL-NEXT: ;;#ASMSTART
; CHECK-GISEL-NEXT: ; use s[4:19]
; CHECK-GISEL-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index 83619ccbbedef..a5893e8acd786 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -5641,19 +5641,35 @@ define half @v_exp_f16_fast(half %in) {
}
define <2 x half> @v_exp_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_exp_v2f16:
; GFX900: ; %bb.0:
@@ -5669,21 +5685,39 @@ define <2 x half> @v_exp_v2f16(<2 x half> %in) {
; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_v2f16:
; R600: ; %bb.0:
@@ -5699,19 +5733,36 @@ define <2 x half> @v_exp_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp_fabs_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; VI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5742,21 +5793,40 @@ define <2 x half> @v_exp_fabs_v2f16(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_fabs_v2f16:
; R600: ; %bb.0:
@@ -5773,19 +5843,36 @@ define <2 x half> @v_exp_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp_fneg_fabs_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp_fneg_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; VI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_fneg_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_fneg_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_fneg_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5816,22 +5903,41 @@ define <2 x half> @v_exp_fneg_fabs_v2f16(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_fneg_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_fneg_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_fneg_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_fneg_fabs_v2f16:
; R600: ; %bb.0:
@@ -5849,19 +5955,36 @@ define <2 x half> @v_exp_fneg_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp_fneg_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp_fneg_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_fneg_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_fneg_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_fneg_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5892,22 +6015,41 @@ define <2 x half> @v_exp_fneg_v2f16(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_fneg_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_fneg_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_fneg_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_fneg_v2f16:
; R600: ; %bb.0:
@@ -5924,16 +6066,29 @@ define <2 x half> @v_exp_fneg_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp_v2f16_fast(<2 x half> %in) {
-; VI-LABEL: v_exp_v2f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, 0x3dc5
-; VI-NEXT: v_mul_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x3dc5, v0
-; VI-NEXT: v_exp_f16_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_v2f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x3dc5
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x3dc5, v0
+; VI-SDAG-NEXT: v_exp_f16_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_v2f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x3dc5
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x3dc5, v0
+; VI-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f16_e32 v1, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_v2f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -5956,21 +6111,43 @@ define <2 x half> @v_exp_v2f16_fast(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_v2f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8a000, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v2f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8a000, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v2f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8a000, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_v2f16_fast:
; R600: ; %bb.0:
@@ -5986,23 +6163,44 @@ define <2 x half> @v_exp_v2f16_fast(<2 x half> %in) {
}
define <3 x half> @v_exp_v3f16(<3 x half> %in) {
-; VI-LABEL: v_exp_v3f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
-; VI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; VI-NEXT: v_exp_f32_e32 v2, v2
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_v3f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_v3f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; VI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_v3f16:
; GFX900-SDAG: ; %bb.0:
@@ -6041,25 +6239,48 @@ define <3 x half> @v_exp_v3f16(<3 x half> %in) {
; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_v3f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v3f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v3f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3fb8aa3b, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8aa3b, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8aa3b, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_v3f16:
; R600: ; %bb.0:
@@ -6075,18 +6296,34 @@ define <3 x half> @v_exp_v3f16(<3 x half> %in) {
}
define <3 x half> @v_exp_v3f16_afn(<3 x half> %in) {
-; VI-LABEL: v_exp_v3f16_afn:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, 0x3dc5
-; VI-NEXT: v_mul_f16_e32 v2, 0x3dc5, v0
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_exp_f16_e32 v2, v2
-; VI-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v1, 0x3dc5, v1
-; VI-NEXT: v_exp_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp_v3f16_afn:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x3dc5
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x3dc5, v0
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_exp_f16_e32 v2, v2
+; VI-SDAG-NEXT: v_exp_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, 0x3dc5, v1
+; VI-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_v3f16_afn:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3dc5
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, 0x3dc5, v0
+; VI-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x3dc5, v1
+; VI-GISEL-NEXT: v_exp_f16_e32 v3, v3
+; VI-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_v3f16_afn:
; GFX900-SDAG: ; %bb.0:
@@ -6115,31 +6352,60 @@ define <3 x half> @v_exp_v3f16_afn(<3 x half> %in) {
; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp_v3f16_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3fb8a000, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3fb8a000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v3f16_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3fb8a000, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3fb8a000, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp_v3f16_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3fb8a000, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3fb8a000, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp_v3f16_afn:
; R600: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index a9ce3a7130396..8c7af7cab8c7f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -5727,19 +5727,35 @@ define half @v_exp10_f16_fast(half %in) {
}
define <2 x half> @v_exp10_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp10_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_exp10_v2f16:
; GFX900: ; %bb.0:
@@ -5755,21 +5771,39 @@ define <2 x half> @v_exp10_v2f16(<2 x half> %in) {
; GFX900-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_v2f16:
; R600: ; %bb.0:
@@ -5785,19 +5819,36 @@ define <2 x half> @v_exp10_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp10_fabs_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp10_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; VI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp10_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5828,21 +5879,40 @@ define <2 x half> @v_exp10_fabs_v2f16(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_fabs_v2f16:
; R600: ; %bb.0:
@@ -5859,19 +5929,36 @@ define <2 x half> @v_exp10_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp10_fneg_fabs_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp10_fneg_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; VI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_fneg_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_fneg_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp10_fneg_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5902,22 +5989,41 @@ define <2 x half> @v_exp10_fneg_fabs_v2f16(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_fneg_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_fneg_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_fneg_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_fneg_fabs_v2f16:
; R600: ; %bb.0:
@@ -5935,19 +6041,36 @@ define <2 x half> @v_exp10_fneg_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp10_fneg_v2f16(<2 x half> %in) {
-; VI-LABEL: v_exp10_fneg_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_fneg_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_fneg_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp10_fneg_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5978,22 +6101,41 @@ define <2 x half> @v_exp10_fneg_v2f16(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v1, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_fneg_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_fneg_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_fneg_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_fneg_v2f16:
; R600: ; %bb.0:
@@ -6010,23 +6152,43 @@ define <2 x half> @v_exp10_fneg_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp10_v2f16_fast(<2 x half> %in) {
-; VI-LABEL: v_exp10_v2f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, 0x113c
-; VI-NEXT: v_mov_b32_e32 v2, 0x42a4
-; VI-NEXT: v_mul_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v3, 0x113c, v0
-; VI-NEXT: v_mul_f16_e32 v0, 0x42a4, v0
-; VI-NEXT: v_exp_f16_e32 v1, v1
-; VI-NEXT: v_exp_f16_e32 v2, v2
-; VI-NEXT: v_exp_f16_e32 v3, v3
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_mul_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, v0, v3
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_v2f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x113c
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42a4
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v3, 0x113c, v0
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x42a4, v0
+; VI-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f16_e32 v2, v2
+; VI-SDAG-NEXT: v_exp_f16_e32 v3, v3
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_v2f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 0x113c
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 0x42a4
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x113c, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v4, 0x42a4, v0
+; VI-GISEL-NEXT: v_mul_f16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f16_e32 v2, v2
+; VI-GISEL-NEXT: v_exp_f16_e32 v4, v4
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, v0, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, v4, v2
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp10_v2f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -6064,27 +6226,67 @@ define <2 x half> @v_exp10_v2f16_fast(<2 x half> %in) {
; GFX900-GISEL-NEXT: v_pack_b32_f16 v0, v2, v0
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_v2f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3a278000, v1
-; SI-NEXT: v_mul_f32_e32 v1, 0x40548000, v1
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v3, 0x3a278000, v0
-; SI-NEXT: v_mul_f32_e32 v0, 0x40548000, v0
-; SI-NEXT: v_exp_f32_e32 v3, v3
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, v1, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, v0, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v2f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3a278000, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40548000, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3a278000, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40548000, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, v1, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v2f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3a278000, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40548000, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3a278000, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40548000, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, v0, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, v1, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_v2f16_fast:
; R600: ; %bb.0:
@@ -6100,23 +6302,44 @@ define <2 x half> @v_exp10_v2f16_fast(<2 x half> %in) {
}
define <3 x half> @v_exp10_v3f16(<3 x half> %in) {
-; VI-LABEL: v_exp10_v3f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
-; VI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; VI-NEXT: v_exp_f32_e32 v2, v2
-; VI-NEXT: v_exp_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; VI-NEXT: v_exp_f32_e32 v1, v1
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_v3f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_v3f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; VI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; VI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp10_v3f16:
; GFX900-SDAG: ; %bb.0:
@@ -6155,25 +6378,48 @@ define <3 x half> @v_exp10_v3f16(<3 x half> %in) {
; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_v3f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v3f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v3f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x40549a78, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40549a78, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40549a78, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_v3f16:
; R600: ; %bb.0:
@@ -6189,28 +6435,54 @@ define <3 x half> @v_exp10_v3f16(<3 x half> %in) {
}
define <3 x half> @v_exp10_v3f16_afn(<3 x half> %in) {
-; VI-LABEL: v_exp10_v3f16_afn:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v5, 0x113c
-; VI-NEXT: v_mov_b32_e32 v6, 0x42a4
-; VI-NEXT: v_mul_f16_e32 v2, 0x113c, v1
-; VI-NEXT: v_mul_f16_e32 v1, 0x42a4, v1
-; VI-NEXT: v_mul_f16_e32 v3, 0x113c, v0
-; VI-NEXT: v_mul_f16_e32 v4, 0x42a4, v0
-; VI-NEXT: v_mul_f16_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_exp_f16_e32 v2, v2
-; VI-NEXT: v_exp_f16_e32 v1, v1
-; VI-NEXT: v_exp_f16_e32 v3, v3
-; VI-NEXT: v_exp_f16_e32 v4, v4
-; VI-NEXT: v_exp_f16_e32 v5, v5
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_mul_f16_e32 v1, v1, v2
-; VI-NEXT: v_mul_f16_e32 v2, v4, v3
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; VI-SDAG-LABEL: v_exp10_v3f16_afn:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v5, 0x113c
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, 0x42a4
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x113c, v1
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, 0x42a4, v1
+; VI-SDAG-NEXT: v_mul_f16_e32 v3, 0x113c, v0
+; VI-SDAG-NEXT: v_mul_f16_e32 v4, 0x42a4, v0
+; VI-SDAG-NEXT: v_mul_f16_sdwa v5, v0, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_exp_f16_e32 v2, v2
+; VI-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_exp_f16_e32 v3, v3
+; VI-SDAG-NEXT: v_exp_f16_e32 v4, v4
+; VI-SDAG-NEXT: v_exp_f16_e32 v5, v5
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, v1, v2
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, v4, v3
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp10_v3f16_afn:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x113c
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x42a4
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, 0x113c, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v5, 0x42a4, v0
+; VI-GISEL-NEXT: v_mul_f16_sdwa v2, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_exp_f16_e32 v2, v2
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v4, 0x113c, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x42a4, v1
+; VI-GISEL-NEXT: v_exp_f16_e32 v3, v3
+; VI-GISEL-NEXT: v_exp_f16_e32 v5, v5
+; VI-GISEL-NEXT: v_exp_f16_e32 v4, v4
+; VI-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, v5, v3
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, v1, v4
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp10_v3f16_afn:
; GFX900-SDAG: ; %bb.0:
@@ -6259,58 +6531,114 @@ define <3 x half> @v_exp10_v3f16_afn(<3 x half> %in) {
; GFX900-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; SI-LABEL: v_exp10_v3f16_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v5, 0x3a278000, v2
-; SI-NEXT: v_mul_f32_e32 v2, 0x40548000, v2
-; SI-NEXT: v_mul_f32_e32 v4, 0x3a278000, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x40548000, v0
-; SI-NEXT: v_mul_f32_e32 v3, 0x3a278000, v1
-; SI-NEXT: v_mul_f32_e32 v1, 0x40548000, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v5, v5
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v4, v4
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v3, v3
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v5, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v4, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, v2, v5
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v0, v0, v4
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, v1, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp10_v3f16_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3a278000, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x40548000, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3a278000, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x40548000, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3a278000, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40548000, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v5, v5
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v4, v4
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, v2, v5
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, v0, v4
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, v1, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; SI-GISEL-LABEL: v_exp10_v3f16_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3a278000, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x40548000, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v5, 0x3a278000, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x40548000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v5, v5
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v4, 0x3a278000, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x40548000, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, v2, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v5
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v4, v4
+; SI-GISEL-NEXT: v_exp_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, v1, v4
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, v0, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; R600-LABEL: v_exp10_v3f16_afn:
; R600: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 6268eec23a7c7..f89b2b936e957 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -3072,27 +3072,53 @@ define half @v_exp2_f16_fast(half %in) {
}
define <2 x half> @v_exp2_v2f16(<2 x half> %in) {
-; SI-LABEL: v_exp2_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp2_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp2_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp2_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp2_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -3124,27 +3150,55 @@ define <2 x half> @v_exp2_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp2_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_exp2_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp2_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v1, |v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e64 v0, |v0|
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp2_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v1, |v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e64 v0, |v0|
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp2_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-GISEL-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp2_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -3178,28 +3232,56 @@ define <2 x half> @v_exp2_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp2_fneg_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_exp2_fneg_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fneg_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp2_fneg_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v1, -|v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e64 v0, -|v0|
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp2_fneg_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_fneg_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v1, -|v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e64 v0, -|v0|
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp2_fneg_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp2_fneg_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -3234,28 +3316,56 @@ define <2 x half> @v_exp2_fneg_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp2_fneg_v2f16(<2 x half> %in) {
-; SI-LABEL: v_exp2_fneg_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_fneg_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp2_fneg_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v1, -v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e64 v0, -v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp2_fneg_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_fneg_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v1, -v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e64 v0, -v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp2_fneg_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp2_fneg_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -3289,27 +3399,53 @@ define <2 x half> @v_exp2_fneg_v2f16(<2 x half> %in) {
}
define <2 x half> @v_exp2_v2f16_fast(<2 x half> %in) {
-; SI-LABEL: v_exp2_v2f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_v2f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp2_v2f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp2_v2f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_v2f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp2_v2f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_exp_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp2_v2f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -3341,31 +3477,63 @@ define <2 x half> @v_exp2_v2f16_fast(<2 x half> %in) {
}
define <3 x half> @v_exp_v3f16(<3 x half> %in) {
-; SI-LABEL: v_exp_v3f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp_v3f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp_v3f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_exp_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp_v3f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp_v3f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp_v3f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp_v3f16:
; GFX900-SDAG: ; %bb.0:
@@ -3400,31 +3568,63 @@ define <3 x half> @v_exp_v3f16(<3 x half> %in) {
}
define <3 x half> @v_exp2_v3f16_afn(<3 x half> %in) {
-; SI-LABEL: v_exp2_v3f16_afn:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_exp_f32_e32 v2, v2
-; SI-NEXT: v_exp_f32_e32 v0, v0
-; SI-NEXT: v_exp_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_exp2_v3f16_afn:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_exp_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_exp2_v3f16_afn:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_exp_f16_e32 v0, v0
-; VI-NEXT: v_exp_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_exp2_v3f16_afn:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_exp_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_exp_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_exp_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_exp2_v3f16_afn:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_exp_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_exp_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_exp2_v3f16_afn:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_exp_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_exp_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_exp_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_exp2_v3f16_afn:
; GFX900-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
index ce5426b4efd8e..92219c51d5e15 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.frexp.ll
@@ -7,13 +7,13 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define { half, i32 } @test_frexp_f16_i32(half %a) {
; GFX6-SDAG-LABEL: test_frexp_f16_i32:
@@ -380,41 +380,41 @@ define i32 @test_frexp_f16_i32_only_use_exp(half %a) {
}
define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
-; GFX6-LABEL: test_frexp_v2f16_v2i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v0
-; GFX6-NEXT: s_mov_b32 s4, 0x7f800000
-; GFX6-NEXT: v_frexp_mant_f32_e32 v0, v1
-; GFX6-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX6-NEXT: v_frexp_mant_f32_e32 v2, v3
-; GFX6-NEXT: v_cmp_lt_f32_e64 s[4:5], |v3|, s4
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[4:5]
-; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; GFX6-NEXT: v_frexp_exp_i32_f32_e32 v1, v3
-; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_frexp_v2f16_v2i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_frexp_mant_f16_e32 v1, v0
-; GFX8-NEXT: v_frexp_mant_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v3, v1, v2
-; GFX8-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
-; GFX8-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX8-NEXT: v_bfe_i32 v1, v0, 0, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, v3
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_frexp_v2f16_v2i32:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX6-SDAG-NEXT: v_frexp_mant_f32_e32 v0, v1
+; GFX6-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; GFX6-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
+; GFX6-SDAG-NEXT: v_frexp_mant_f32_e32 v2, v3
+; GFX6-SDAG-NEXT: v_cmp_lt_f32_e64 s[4:5], |v3|, s4
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cndmask_b32_e64 v2, v3, v2, s[4:5]
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v1, v3
+; GFX6-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_frexp_v2f16_v2i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_frexp_mant_f16_e32 v1, v0
+; GFX8-SDAG-NEXT: v_frexp_mant_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v3, v1, v2
+; GFX8-SDAG-NEXT: v_frexp_exp_i16_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-SDAG-NEXT: v_frexp_exp_i16_f16_e32 v0, v0
+; GFX8-SDAG-NEXT: v_bfe_i32 v2, v1, 0, 16
+; GFX8-SDAG-NEXT: v_bfe_i32 v1, v0, 0, 16
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, v3
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_frexp_v2f16_v2i32:
; GFX9-SDAG: ; %bb.0:
@@ -491,6 +491,45 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
; GFX12-SDAG-FAKE16-NEXT: v_bfe_i32 v2, v4, 0, 16
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_frexp_v2f16_v2i32:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v3, 0x7f800000
+; GFX6-GISEL-NEXT: v_frexp_mant_f32_e32 v4, v2
+; GFX6-GISEL-NEXT: v_frexp_exp_i32_f32_e32 v1, v2
+; GFX6-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v2|, v3
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX6-GISEL-NEXT: v_frexp_mant_f32_e32 v5, v0
+; GFX6-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v3
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v4, v2
+; GFX6-GISEL-NEXT: v_frexp_exp_i32_f32_e32 v2, v0
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_bfe_u32 v3, v4, 0, 16
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v3, v0
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_frexp_v2f16_v2i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_frexp_mant_f16_e32 v3, v0
+; GFX8-GISEL-NEXT: v_frexp_exp_i16_f16_e32 v1, v0
+; GFX8-GISEL-NEXT: v_frexp_mant_f16_sdwa v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-GISEL-NEXT: v_frexp_exp_i16_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-GISEL-NEXT: v_bfe_i32 v2, v0, 0, 16
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_frexp_v2f16_v2i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -569,32 +608,32 @@ define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) {
}
define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
-; GFX6-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s4, 0x7f800000
-; GFX6-NEXT: v_frexp_mant_f32_e32 v2, v1
-; GFX6-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
-; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT: v_frexp_mant_f32_e32 v2, v0
-; GFX6-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_frexp_mant_f16_e32 v1, v0
-; GFX8-NEXT: v_frexp_mant_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX6-SDAG-NEXT: v_frexp_mant_f32_e32 v2, v1
+; GFX6-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; GFX6-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
+; GFX6-SDAG-NEXT: v_frexp_mant_f32_e32 v2, v0
+; GFX6-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, s4
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_frexp_mant_f16_e32 v1, v0
+; GFX8-SDAG-NEXT: v_frexp_mant_f16_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
; GFX9-SDAG: ; %bb.0:
@@ -646,6 +685,37 @@ define <2 x half> @test_frexp_v2f16_v2i32_only_use_fract(<2 x half> %a) {
; GFX12-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_mov_b32_e32 v2, 0x7f800000
+; GFX6-GISEL-NEXT: v_frexp_mant_f32_e32 v3, v1
+; GFX6-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, v2
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX6-GISEL-NEXT: v_frexp_mant_f32_e32 v3, v0
+; GFX6-GISEL-NEXT: v_cmp_lt_f32_e64 vcc, |v0|, v2
+; GFX6-GISEL-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_frexp_mant_f16_e32 v1, v0
+; GFX8-GISEL-NEXT: v_frexp_mant_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_frexp_v2f16_v2i32_only_use_fract:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -2062,3 +2132,4 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
; GFX11-SDAG: {{.*}}
; GFX12-GISEL: {{.*}}
; GFX12-SDAG: {{.*}}
+; GFX6: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
index 2502760abb912..2029e3c9b3db9 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.ldexp.ll
@@ -5,11 +5,11 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG-FAKE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL-FAKE16 %s
define float @test_ldexp_f32_i32(ptr addrspace(1) %out, float %a, i32 %b) {
; GFX6-LABEL: test_ldexp_f32_i32:
@@ -406,31 +406,31 @@ define half @test_ldexp_f16_i32(half %a, i32 %b) {
}
define <2 x half> @test_ldexp_v2f16_v2i32(<2 x half> %a, <2 x i32> %b) {
-; GFX6-LABEL: test_ldexp_v2f16_v2i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_ldexp_f32_e32 v2, v3, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v1
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_ldexp_v2f16_v2i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v3, 0x7fff
-; GFX8-NEXT: v_med3_i32 v2, v2, s4, v3
-; GFX8-NEXT: v_med3_i32 v1, v1, s4, v3
-; GFX8-NEXT: v_ldexp_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_ldexp_v2f16_v2i32:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v2, v3, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_ldexp_v2f16_v2i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v3, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v2, v2, s4, v3
+; GFX8-SDAG-NEXT: v_med3_i32 v1, v1, s4, v3
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v2, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v2f16_v2i32:
; GFX9-SDAG: ; %bb.0:
@@ -470,6 +470,36 @@ define <2 x half> @test_ldexp_v2f16_v2i32(<2 x half> %a, <2 x i32> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_ldexp_v2f16_v2i32:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v1, v3, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v2
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_ldexp_v2f16_v2i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v1, v1, v3, v4
+; GFX8-GISEL-NEXT: v_med3_i32 v2, v2, v3, v4
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v1, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v2f16_v2i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -514,29 +544,29 @@ define <2 x half> @test_ldexp_v2f16_v2i32(<2 x half> %a, <2 x i32> %b) {
}
define <2 x half> @test_ldexp_v2f16_v2i16(<2 x half> %a, <2 x i16> %b) {
-; GFX6-LABEL: test_ldexp_v2f16_v2i16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_ashrrev_i32_e32 v3, 16, v1
-; GFX6-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX6-NEXT: v_ldexp_f32_e32 v2, v2, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v1
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v2
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_ldexp_v2f16_v2i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ldexp_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_ldexp_v2f16_v2i16:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_ashrrev_i32_e32 v3, 16, v1
+; GFX6-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v2, v2, v3
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v2
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_ldexp_v2f16_v2i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v2f16_v2i16:
; GFX9-SDAG: ; %bb.0:
@@ -564,6 +594,34 @@ define <2 x half> @test_ldexp_v2f16_v2i16(<2 x half> %a, <2 x i16> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_ldexp_v2f16_v2i16:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_bfe_i32 v3, v1, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_i32 v1, v1, 16, 16
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v2, v2, v3
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v1
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_ldexp_v2f16_v2i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v2f16_v2i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -596,36 +654,36 @@ define <2 x half> @test_ldexp_v2f16_v2i16(<2 x half> %a, <2 x i16> %b) {
}
define <3 x half> @test_ldexp_v3f16_v3i32(<3 x half> %a, <3 x i32> %b) {
-; GFX6-LABEL: test_ldexp_v3f16_v3i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_ldexp_f32_e32 v3, v5, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_ldexp_f32_e32 v1, v1, v4
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_ldexp_v3f16_v3i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v5, 0x7fff
-; GFX8-NEXT: v_med3_i32 v2, v2, s4, v5
-; GFX8-NEXT: v_med3_i32 v3, v3, s4, v5
-; GFX8-NEXT: v_med3_i32 v4, v4, s4, v5
-; GFX8-NEXT: v_ldexp_f16_e32 v2, v0, v2
-; GFX8-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_ldexp_f16_e32 v1, v1, v4
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_ldexp_v3f16_v3i32:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v3, v5, v3
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v4
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_ldexp_v3f16_v3i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v5, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v2, v2, s4, v5
+; GFX8-SDAG-NEXT: v_med3_i32 v3, v3, s4, v5
+; GFX8-SDAG-NEXT: v_med3_i32 v4, v4, s4, v5
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v2, v0, v2
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v4
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v3f16_v3i32:
; GFX9-SDAG: ; %bb.0:
@@ -671,6 +729,43 @@ define <3 x half> @test_ldexp_v3f16_v3i32(<3 x half> %a, <3 x i32> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v1, v1, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_ldexp_v3f16_v3i32:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v5, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v2, v5, v2
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v3
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v4
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_ldexp_v3f16_v3i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v6, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v2, v2, v5, v6
+; GFX8-GISEL-NEXT: v_med3_i32 v3, v3, v5, v6
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v2, v0, v2
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_med3_i32 v3, v4, v5, v6
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v3f16_v3i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -725,34 +820,34 @@ define <3 x half> @test_ldexp_v3f16_v3i32(<3 x half> %a, <3 x i32> %b) {
}
define <3 x half> @test_ldexp_v3f16_v3i16(<3 x half> %a, <3 x i16> %b) {
-; GFX6-LABEL: test_ldexp_v3f16_v3i16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v4, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_ashrrev_i32_e32 v5, 16, v2
-; GFX6-NEXT: v_ldexp_f32_e32 v4, v4, v5
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_ldexp_f32_e32 v1, v1, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_ldexp_v3f16_v3i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ldexp_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_ldexp_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_ldexp_v3f16_v3i16:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_ashrrev_i32_e32 v5, 16, v2
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v4, v4, v5
+; GFX6-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v4
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_ldexp_v3f16_v3i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v3f16_v3i16:
; GFX9-SDAG: ; %bb.0:
@@ -783,6 +878,41 @@ define <3 x half> @test_ldexp_v3f16_v3i16(<3 x half> %a, <3 x i16> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_ldexp_v3f16_v3i16:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_bfe_i32 v5, v2, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v2
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v4, v4, v5
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_bfe_i32 v2, v3, 0, 16
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v2
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v2, v4, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_ldexp_v3f16_v3i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v3f16_v3i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -822,45 +952,45 @@ define <3 x half> @test_ldexp_v3f16_v3i16(<3 x half> %a, <3 x i16> %b) {
}
define <4 x half> @test_ldexp_v4f16_v4i32(<4 x half> %a, <4 x i32> %b) {
-; GFX6-LABEL: test_ldexp_v4f16_v4i32:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GFX6-NEXT: v_lshrrev_b32_e32 v7, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_ldexp_f32_e32 v5, v6, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v7
-; GFX6-NEXT: v_cvt_f16_f32_e32 v5, v5
-; GFX6-NEXT: v_ldexp_f32_e32 v1, v1, v4
-; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
-; GFX6-NEXT: v_ldexp_f32_e32 v3, v6, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v5
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_ldexp_v4f16_v4i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v6, 0x7fff
-; GFX8-NEXT: v_med3_i32 v4, v4, s4, v6
-; GFX8-NEXT: v_med3_i32 v5, v5, s4, v6
-; GFX8-NEXT: v_med3_i32 v2, v2, s4, v6
-; GFX8-NEXT: v_med3_i32 v3, v3, s4, v6
-; GFX8-NEXT: v_ldexp_f16_e32 v4, v1, v4
-; GFX8-NEXT: v_ldexp_f16_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_ldexp_f16_e32 v2, v0, v2
-; GFX8-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v1, v4, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_ldexp_v4f16_v4i32:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v5, v6, v5
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v6, v7
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v5, v5
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v4
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v3, v6, v3
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_ldexp_v4f16_v4i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v6, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v4, v4, s4, v6
+; GFX8-SDAG-NEXT: v_med3_i32 v5, v5, s4, v6
+; GFX8-SDAG-NEXT: v_med3_i32 v2, v2, s4, v6
+; GFX8-SDAG-NEXT: v_med3_i32 v3, v3, s4, v6
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v4, v1, v4
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v1, v1, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v2, v0, v2
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v4, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v4f16_v4i32:
; GFX9-SDAG: ; %bb.0:
@@ -917,6 +1047,53 @@ define <4 x half> @test_ldexp_v4f16_v4i32(<4 x half> %a, <4 x i32> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v5
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_ldexp_v4f16_v4i32:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v2, v6, v2
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v3
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v7
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v4
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v3, v3, v5
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_ldexp_v4f16_v4i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v6, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v7, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v2, v2, v6, v7
+; GFX8-GISEL-NEXT: v_med3_i32 v3, v3, v6, v7
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v2, v0, v2
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_med3_i32 v3, v4, v6, v7
+; GFX8-GISEL-NEXT: v_med3_i32 v4, v5, v6, v7
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v3, v1, v3
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v4f16_v4i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -980,43 +1157,43 @@ define <4 x half> @test_ldexp_v4f16_v4i32(<4 x half> %a, <4 x i32> %b) {
}
define <4 x half> @test_ldexp_v4f16_v4i16(<4 x half> %a, <4 x i16> %b) {
-; GFX6-LABEL: test_ldexp_v4f16_v4i16:
-; GFX6: ; %bb.0:
-; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GFX6-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GFX6-NEXT: v_ashrrev_i32_e32 v4, 16, v2
-; GFX6-NEXT: v_ashrrev_i32_e32 v7, 16, v3
-; GFX6-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX6-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX6-NEXT: v_ldexp_f32_e32 v6, v6, v7
-; GFX6-NEXT: v_ldexp_f32_e32 v4, v5, v4
-; GFX6-NEXT: v_cvt_f16_f32_e32 v6, v6
-; GFX6-NEXT: v_cvt_f16_f32_e32 v4, v4
-; GFX6-NEXT: v_ldexp_f32_e32 v1, v1, v3
-; GFX6-NEXT: v_ldexp_f32_e32 v0, v0, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX8-LABEL: test_ldexp_v4f16_v4i16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_ldexp_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ldexp_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_ldexp_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX6-SDAG-LABEL: test_ldexp_v4f16_v4i16:
+; GFX6-SDAG: ; %bb.0:
+; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v5, 16, v0
+; GFX6-SDAG-NEXT: v_lshrrev_b32_e32 v6, 16, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-SDAG-NEXT: v_ashrrev_i32_e32 v4, 16, v2
+; GFX6-SDAG-NEXT: v_ashrrev_i32_e32 v7, 16, v3
+; GFX6-SDAG-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX6-SDAG-NEXT: v_bfe_i32 v3, v3, 0, 16
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v6, v6, v7
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v4, v5, v4
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v6, v6
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v4, v4
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v1, v1, v3
+; GFX6-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v2
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX6-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; GFX6-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_ldexp_v4f16_v4i16:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v4f16_v4i16:
; GFX9-SDAG: ; %bb.0:
@@ -1055,6 +1232,51 @@ define <4 x half> @test_ldexp_v4f16_v4i16(<4 x half> %a, <4 x i16> %b) {
; GFX11-SDAG-FAKE16-NEXT: v_pack_b32_f16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX6-GISEL-LABEL: test_ldexp_v4f16_v4i16:
+; GFX6-GISEL: ; %bb.0:
+; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v4, 16, v0
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-GISEL-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GFX6-GISEL-NEXT: v_bfe_i32 v6, v2, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_i32 v2, v2, 16, 16
+; GFX6-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v2, v4, v2
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v0, v0, v6
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX6-GISEL-NEXT: v_bfe_i32 v4, v3, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_i32 v3, v3, 16, 16
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v3, v5, v3
+; GFX6-GISEL-NEXT: v_ldexp_f32_e32 v1, v1, v4
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX6-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX6-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GFX6-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_ldexp_v4f16_v4i16:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v4f16_v4i16:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -1222,6 +1444,3 @@ declare double @llvm.ldexp.f64.i32(double, i32) #0
declare <2 x double> @llvm.ldexp.v2f64.v2i32(<2 x double>, <2 x i32>) #0
attributes #0 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX6-GISEL: {{.*}}
-; GFX6-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index b0623c84528a3..3fcec611b218b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -4585,35 +4585,69 @@ define half @v_log_f16_fast(half %in) {
}
define <2 x half> @v_log_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_log_v2f16:
; GFX900: ; %bb.0:
@@ -4704,35 +4738,71 @@ define <2 x half> @v_log_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -4843,36 +4913,72 @@ define <2 x half> @v_log_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log_fneg_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log_fneg_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_fneg_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_fneg_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_fneg_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_fneg_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_fneg_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_fneg_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -4984,36 +5090,72 @@ define <2 x half> @v_log_fneg_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log_fneg_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log_fneg_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_fneg_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_fneg_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_fneg_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_fneg_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_fneg_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_fneg_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5124,32 +5266,62 @@ define <2 x half> @v_log_fneg_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log_v2f16_fast(<2 x half> %in) {
-; SI-LABEL: v_log_v2f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v2f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v2f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_mov_b32_e32 v2, 0x398c
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x398c, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v2f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v2f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x398c
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v2f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_log_v2f16_fast:
; GFX900: ; %bb.0:
@@ -5221,43 +5393,87 @@ define <2 x half> @v_log_v2f16_fast(<2 x half> %in) {
}
define <3 x half> @v_log_v3f16(<3 x half> %in) {
-; SI-LABEL: v_log_v3f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v3f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v3f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v2, v2
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; VI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v3f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v3f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v3f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v3f16:
; GFX900-SDAG: ; %bb.0:
@@ -5390,38 +5606,76 @@ define <3 x half> @v_log_v3f16(<3 x half> %in) {
}
define <3 x half> @v_log_v3f16_fast(<3 x half> %in) {
-; SI-LABEL: v_log_v3f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v3f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v3f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v0
-; VI-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_mov_b32_e32 v3, 0x398c
-; VI-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v1, 0x398c, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v3f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v3f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x398c
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v3f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v3f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -5522,55 +5776,112 @@ define <3 x half> @v_log_v3f16_fast(<3 x half> %in) {
}
define <4 x half> @v_log_v4f16(<4 x half> %in) {
-; SI-LABEL: v_log_v4f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; SI-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v4f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v4f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f32_e32 v2, v2
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v3, v3
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; VI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; VI-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
-; VI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v3, v0
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v4f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v4f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v1
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v0
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v3, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v4f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v4f16:
; GFX900-SDAG: ; %bb.0:
@@ -5721,48 +6032,97 @@ define <4 x half> @v_log_v4f16(<4 x half> %in) {
}
define <4 x half> @v_log_v4f16_fast(<4 x half> %in) {
-; SI-LABEL: v_log_v4f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; SI-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v4f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v4f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v1
-; VI-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v3, v0
-; VI-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_mov_b32_e32 v4, 0x398c
-; VI-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v3, 0x398c, v3
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v3, v0
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v4f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v4f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v1
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v3, v0
+; VI-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x398c
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v3, 0x398c, v3
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v3, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v4f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, 0x398c, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v4f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -5949,32 +6309,62 @@ define half @v_log_f16_fpmath(half %x) {
}
define <2 x half> @v_log_v2f16_fpmath(<2 x half> %x) {
-; SI-LABEL: v_log_v2f16_fpmath:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v2f16_fpmath:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v2f16_fpmath:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_mov_b32_e32 v2, 0x398c
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x398c, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v2f16_fpmath:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v2f16_fpmath:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x398c
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v2f16_fpmath:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v2f16_fpmath:
; GFX900-SDAG: ; %bb.0:
@@ -6052,38 +6442,76 @@ define <2 x half> @v_log_v2f16_fpmath(<2 x half> %x) {
}
define <3 x half> @v_log_v3f16_fpmath(<3 x half> %x) {
-; SI-LABEL: v_log_v3f16_fpmath:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v3f16_fpmath:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v3f16_fpmath:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v0
-; VI-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_mov_b32_e32 v3, 0x398c
-; VI-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v1, 0x398c, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v3f16_fpmath:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v3f16_fpmath:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x398c
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v3f16_fpmath:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v3f16_fpmath:
; GFX900-SDAG: ; %bb.0:
@@ -6185,48 +6613,97 @@ define <3 x half> @v_log_v3f16_fpmath(<3 x half> %x) {
}
define <4 x half> @v_log_v4f16_fpmath(<4 x half> %x) {
-; SI-LABEL: v_log_v4f16_fpmath:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
-; SI-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log_v4f16_fpmath:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log_v4f16_fpmath:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v1
-; VI-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0x398c
-; VI-NEXT: v_mul_f16_e32 v2, 0x398c, v2
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x398c, v0
-; VI-NEXT: v_mul_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v0, v3
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log_v4f16_fpmath:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3f317218, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3f317218, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3f317218, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3f317218, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log_v4f16_fpmath:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v1
+; VI-SDAG-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x398c
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-SDAG-NEXT: v_mul_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log_v4f16_fpmath:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x398c, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x398c, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x398c, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, 0x398c, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log_v4f16_fpmath:
; GFX900-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 2f5beb4864b47..fc27dd6f7619b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -4585,35 +4585,69 @@ define half @v_log10_f16_fast(half %in) {
}
define <2 x half> @v_log10_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log10_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_log10_v2f16:
; GFX900: ; %bb.0:
@@ -4704,35 +4738,71 @@ define <2 x half> @v_log10_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log10_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log10_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, |v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -4843,36 +4913,72 @@ define <2 x half> @v_log10_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log10_fneg_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log10_fneg_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_fneg_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_fneg_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_fneg_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_fneg_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -|v0| dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -|v0|
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_fneg_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_fneg_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -4984,36 +5090,72 @@ define <2 x half> @v_log10_fneg_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log10_fneg_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log10_fneg_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_fneg_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_fneg_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e64 v0, -v0
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; VI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_fneg_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_fneg_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, -v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, -v0
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_fneg_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_fneg_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -5124,32 +5266,62 @@ define <2 x half> @v_log10_fneg_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log10_v2f16_fast(<2 x half> %in) {
-; SI-LABEL: v_log10_v2f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v2f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v2f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_mov_b32_e32 v2, 0x34d1
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v2f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v2f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x34d1
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v2f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-LABEL: v_log10_v2f16_fast:
; GFX900: ; %bb.0:
@@ -5221,43 +5393,87 @@ define <2 x half> @v_log10_v2f16_fast(<2 x half> %in) {
}
define <3 x half> @v_log10_v3f16(<3 x half> %in) {
-; SI-LABEL: v_log10_v3f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v3f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v3f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v2, v2
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; VI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v3f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v3f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v3f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v3f16:
; GFX900-SDAG: ; %bb.0:
@@ -5390,38 +5606,76 @@ define <3 x half> @v_log10_v3f16(<3 x half> %in) {
}
define <3 x half> @v_log10_v3f16_fast(<3 x half> %in) {
-; SI-LABEL: v_log10_v3f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v3f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v3f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v0
-; VI-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_mov_b32_e32 v3, 0x34d1
-; VI-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v3f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v3f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x34d1
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v3f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v3f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -5522,55 +5776,112 @@ define <3 x half> @v_log10_v3f16_fast(<3 x half> %in) {
}
define <4 x half> @v_log10_v4f16(<4 x half> %in) {
-; SI-LABEL: v_log10_v4f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; SI-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v4f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v4f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f32_e32 v2, v2
-; VI-NEXT: v_log_f32_e32 v1, v1
-; VI-NEXT: v_log_f32_e32 v3, v3
-; VI-NEXT: v_log_f32_e32 v0, v0
-; VI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; VI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; VI-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
-; VI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; VI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v3, v0
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v4f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v4f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v1
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v0
+; VI-SDAG-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; VI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_cvt_f16_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v3, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v4f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; VI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; VI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; VI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; VI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; VI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v4f16:
; GFX900-SDAG: ; %bb.0:
@@ -5721,48 +6032,97 @@ define <4 x half> @v_log10_v4f16(<4 x half> %in) {
}
define <4 x half> @v_log10_v4f16_fast(<4 x half> %in) {
-; SI-LABEL: v_log10_v4f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; SI-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v4f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v4f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v1
-; VI-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v3, v0
-; VI-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_mov_b32_e32 v4, 0x34d1
-; VI-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v3, v0
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v4f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v4f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v1
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v3, v0
+; VI-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x34d1
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v3, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v4f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v4f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -5949,32 +6309,62 @@ define half @v_log10_f16_fpmath(half %x) {
}
define <2 x half> @v_log10_v2f16_fpmath(<2 x half> %x) {
-; SI-LABEL: v_log10_v2f16_fpmath:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v2f16_fpmath:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v2f16_fpmath:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_mov_b32_e32 v2, 0x34d1
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v2f16_fpmath:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v1, v0
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v2f16_fpmath:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x34d1
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v2f16_fpmath:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v2f16_fpmath:
; GFX900-SDAG: ; %bb.0:
@@ -6052,38 +6442,76 @@ define <2 x half> @v_log10_v2f16_fpmath(<2 x half> %x) {
}
define <3 x half> @v_log10_v3f16_fpmath(<3 x half> %x) {
-; SI-LABEL: v_log10_v3f16_fpmath:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v3f16_fpmath:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v3f16_fpmath:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v0
-; VI-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_mov_b32_e32 v3, 0x34d1
-; VI-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; VI-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v3f16_fpmath:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v3f16_fpmath:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v0
+; VI-SDAG-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x34d1
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v2, v0
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v3f16_fpmath:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v3f16_fpmath:
; GFX900-SDAG: ; %bb.0:
@@ -6185,48 +6613,97 @@ define <3 x half> @v_log10_v3f16_fpmath(<3 x half> %x) {
}
define <4 x half> @v_log10_v4f16_fpmath(<4 x half> %x) {
-; SI-LABEL: v_log10_v4f16_fpmath:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
-; SI-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_or_b32_e32 v0, v0, v3
-; SI-NEXT: v_or_b32_e32 v1, v1, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log10_v4f16_fpmath:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log10_v4f16_fpmath:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_e32 v2, v1
-; VI-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0x34d1
-; VI-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
-; VI-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
-; VI-NEXT: v_mul_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v0, v3
-; VI-NEXT: v_or_b32_e32 v1, v2, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log10_v4f16_fpmath:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_mul_f32_e32 v0, 0x3e9a209b, v0
+; SI-GISEL-NEXT: v_mul_f32_e32 v2, 0x3e9a209b, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_mul_f32_e32 v3, 0x3e9a209b, v3
+; SI-GISEL-NEXT: v_mul_f32_e32 v1, 0x3e9a209b, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log10_v4f16_fpmath:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_e32 v2, v1
+; VI-SDAG-NEXT: v_log_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x34d1
+; VI-SDAG-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-SDAG-NEXT: v_mul_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-SDAG-NEXT: v_mul_f16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v2, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log10_v4f16_fpmath:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_mul_f16_e32 v0, 0x34d1, v0
+; VI-GISEL-NEXT: v_mul_f16_e32 v2, 0x34d1, v2
+; VI-GISEL-NEXT: v_mul_f16_e32 v1, 0x34d1, v1
+; VI-GISEL-NEXT: v_mul_f16_e32 v3, 0x34d1, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log10_v4f16_fpmath:
; GFX900-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index 5efcbfcc443b3..f47bc253a34ad 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -2589,27 +2589,53 @@ define half @v_log2_f16_fast(half %in) {
}
define <2 x half> @v_log2_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log2_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -2677,27 +2703,55 @@ define <2 x half> @v_log2_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log2_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log2_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |v1|
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |v0|
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v1, |v1|
+; SI-SDAG-NEXT: v_cvt_f32_f16_e64 v0, |v0|
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, |v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e64 v0, |v0|
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, |v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e64 v0, |v0|
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_and_b32_e32 v0, 0x7fff7fff, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -2770,28 +2824,56 @@ define <2 x half> @v_log2_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log2_fneg_fabs_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log2_fneg_fabs_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_fneg_fabs_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_fneg_fabs_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, -|v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e64 v0, -|v0|
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_fneg_fabs_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_fneg_fabs_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, -|v0| dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e64 v0, -|v0|
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_fneg_fabs_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_or_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_fneg_fabs_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -2865,28 +2947,56 @@ define <2 x half> @v_log2_fneg_fabs_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log2_fneg_v2f16(<2 x half> %in) {
-; SI-LABEL: v_log2_fneg_v2f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_fneg_v2f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_fneg_v2f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, -v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e64 v0, -v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_fneg_v2f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_fneg_v2f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, -v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e64 v0, -v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_fneg_v2f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_fneg_v2f16:
; GFX900-SDAG: ; %bb.0:
@@ -2959,27 +3069,53 @@ define <2 x half> @v_log2_fneg_v2f16(<2 x half> %in) {
}
define <2 x half> @v_log2_v2f16_fast(<2 x half> %in) {
-; SI-LABEL: v_log2_v2f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_v2f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_v2f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_v2f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_v2f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_v2f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_v2f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -3047,31 +3183,63 @@ define <2 x half> @v_log2_v2f16_fast(<2 x half> %in) {
}
define <3 x half> @v_log2_v3f16(<3 x half> %in) {
-; SI-LABEL: v_log2_v3f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_v3f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_v3f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_v3f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_v3f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_v3f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_v3f16:
; GFX900-SDAG: ; %bb.0:
@@ -3148,31 +3316,63 @@ define <3 x half> @v_log2_v3f16(<3 x half> %in) {
}
define <3 x half> @v_log2_v3f16_fast(<3 x half> %in) {
-; SI-LABEL: v_log2_v3f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_v3f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_v3f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_v3f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_v3f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_v3f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v0, v0
+; VI-GISEL-NEXT: v_log_f16_e32 v1, v1
+; VI-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_v3f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -3249,39 +3449,80 @@ define <3 x half> @v_log2_v3f16_fast(<3 x half> %in) {
}
define <4 x half> @v_log2_v4f16(<4 x half> %in) {
-; SI-LABEL: v_log2_v4f16:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: v_or_b32_e32 v1, v1, v3
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_v4f16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_v4f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_sdwa v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v3
-; VI-NEXT: v_or_b32_e32 v1, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_v4f16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_v4f16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_sdwa v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_v4f16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_v4f16:
; GFX900-SDAG: ; %bb.0:
@@ -3367,39 +3608,80 @@ define <4 x half> @v_log2_v4f16(<4 x half> %in) {
}
define <4 x half> @v_log2_v4f16_fast(<4 x half> %in) {
-; SI-LABEL: v_log2_v4f16_fast:
-; SI: ; %bb.0:
-; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; SI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; SI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; SI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; SI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; SI-NEXT: v_log_f32_e32 v3, v3
-; SI-NEXT: v_log_f32_e32 v2, v2
-; SI-NEXT: v_log_f32_e32 v0, v0
-; SI-NEXT: v_log_f32_e32 v1, v1
-; SI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; SI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; SI-NEXT: v_or_b32_e32 v0, v0, v2
-; SI-NEXT: v_or_b32_e32 v1, v1, v3
-; SI-NEXT: s_setpc_b64 s[30:31]
+; SI-SDAG-LABEL: v_log2_v4f16_fast:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-SDAG-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_log_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_log_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-SDAG-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-SDAG-NEXT: v_or_b32_e32 v1, v1, v3
+; SI-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_log2_v4f16_fast:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_log_f16_sdwa v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_sdwa v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_log_f16_e32 v0, v0
-; VI-NEXT: v_log_f16_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v3
-; VI-NEXT: v_or_b32_e32 v1, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SI-GISEL-LABEL: v_log2_v4f16_fast:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SI-GISEL-NEXT: v_log_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_log_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_log_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_log_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v2, v2
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v3, v3
+; SI-GISEL-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SI-GISEL-NEXT: v_bfe_u32 v2, v2, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v0, v0, v2
+; SI-GISEL-NEXT: v_bfe_u32 v2, v3, 0, 16
+; SI-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; SI-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-SDAG-LABEL: v_log2_v4f16_fast:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-SDAG-NEXT: v_log_f16_sdwa v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_sdwa v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-SDAG-NEXT: v_log_f16_e32 v0, v0
+; VI-SDAG-NEXT: v_log_f16_e32 v1, v1
+; VI-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v1, v1, v2
+; VI-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; VI-GISEL-LABEL: v_log2_v4f16_fast:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-GISEL-NEXT: v_log_f16_e32 v2, v0
+; VI-GISEL-NEXT: v_log_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_log_f16_e32 v3, v1
+; VI-GISEL-NEXT: v_log_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 16
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; VI-GISEL-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX900-SDAG-LABEL: v_log2_v4f16_fast:
; GFX900-SDAG: ; %bb.0:
@@ -3504,5 +3786,3 @@ attributes #2 = { nocallback nofree nosync nounwind speculatable willreturn memo
; GFX1100-SDAG: {{.*}}
; GFX689-GISEL: {{.*}}
; GFX689-SDAG: {{.*}}
-; VI-GISEL: {{.*}}
-; VI-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
index 8041d30871d14..57f33f2fdb4ef 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi.ll
@@ -9,8 +9,8 @@
; XUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GFX9,GISEL-GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GISEL-CI %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 < %s | FileCheck -check-prefixes=VI,GISEL-VI %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01 < %s | FileCheck -check-prefixes=GISEL-CI %s
define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo(half %src0, half %src1, half %src2) #0 {
; GFX11-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_undeflo:
@@ -103,16 +103,16 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo(half %src0, half %s
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, 0x3c00, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_or_b32_e32 v0, 0x3c00, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo:
; SDAG-CI: ; %bb.0:
@@ -135,16 +135,34 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo(half %src0, half %s
; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v3
; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: s_movk_i32 s4, 0x3c00
+; GISEL-VI-NEXT: s_bfe_u32 s4, s4, 0x100000
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_e32 v0, s4, v0
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_constlo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: s_movk_i32 s4, 0x3c00
+; GISEL-CI-NEXT: s_bfe_u32 s4, s4, 0x100000
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v0, 0x3c00, v0
+; GISEL-CI-NEXT: v_or_b32_e32 v0, s4, v0
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext half %src0 to float
%src1.ext = fpext half %src1 to float
@@ -178,16 +196,16 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
; SDAG-CI: ; %bb.0:
@@ -210,6 +228,19 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, v3
; GISEL-GFX11-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_mov_b32_e32 v1, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -219,6 +250,7 @@ define <2 x half> @v_mad_mixhi_f16_f16lo_f16lo_f16lo_reglo(half %src0, half %src
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
; GISEL-CI-NEXT: v_and_b32_e32 v1, 0xffff, v3
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; GISEL-CI-NEXT: v_or_b32_e32 v0, v1, v0
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
@@ -720,3 +752,4 @@ attributes #0 = { nounwind denormal_fpenv(float: preservesign) }
attributes #1 = { nounwind readnone speculatable }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GISEL-GFX11-FAKE16: {{.*}}
+; VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
index 3596c8d9aaa7e..e7ee371aa5d30 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-lo.ll
@@ -451,21 +451,21 @@ define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half
; GFX906-NEXT: v_mov_b32_e32 v0, v3
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v5, v3, v4
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v2
-; VI-NEXT: v_or_b32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v2
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v2f32:
; SDAG-CI: ; %bb.0:
@@ -496,24 +496,44 @@ define <2 x half> @v_mad_mix_v2f32(<2 x half> %src0, <2 x half> %src1, <2 x half
; GISEL-GFX1100-NEXT: v_mov_b32_e32 v0, v3
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v2f32:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v5
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v2f32:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_mac_f32_e32 v3, v5, v4
+; GISEL-CI-NEXT: v_mac_f32_e32 v5, v3, v4
; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v3
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v2
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v1, v0
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v5
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <2 x half> %src0 to <2 x float>
%src1.ext = fpext <2 x half> %src1 to <2 x float>
@@ -562,26 +582,26 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v3
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v3f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; VI-NEXT: v_mac_f32_e32 v8, v6, v7
-; VI-NEXT: v_mac_f32_e32 v4, v0, v2
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v4
-; VI-NEXT: v_mac_f32_e32 v5, v1, v3
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v5
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v3f32:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-VI-NEXT: v_mac_f32_e32 v8, v6, v7
+; SDAG-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v2, v4
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v5
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v2, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v3f32:
; SDAG-CI: ; %bb.0:
@@ -642,29 +662,56 @@ define <3 x half> @v_mad_mix_v3f32(<3 x half> %src0, <3 x half> %src1, <3 x half
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v6
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v3f32:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v7, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v8, v4
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GISEL-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v8, v6, v7
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v4
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v8
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v2, v5
+; GISEL-VI-NEXT: v_mov_b32_e32 v3, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v3f32:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_mac_f32_e32 v6, v8, v7
+; GISEL-CI-NEXT: v_mac_f32_e32 v8, v6, v7
; GISEL-CI-NEXT: v_mac_f32_e32 v4, v0, v2
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v6
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v4
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v8
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v4
; GISEL-CI-NEXT: v_mac_f32_e32 v5, v1, v3
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v5
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v2, v0
+; GISEL-CI-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <3 x half> %src0 to <3 x float>
%src1.ext = fpext <3 x half> %src1 to <3 x float>
@@ -720,32 +767,32 @@ define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half
; SDAG-GFX906-NEXT: v_mov_b32_e32 v1, v6
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v4f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_cvt_f32_f16_sdwa v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; VI-NEXT: v_mac_f32_e32 v10, v7, v9
-; VI-NEXT: v_mac_f32_e32 v11, v6, v8
-; VI-NEXT: v_mac_f32_e32 v4, v0, v2
-; VI-NEXT: v_mac_f32_e32 v5, v1, v3
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v4
-; VI-NEXT: v_cvt_f16_f32_e32 v3, v5
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: v_or_b32_e32 v1, v3, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v4f32:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-VI-NEXT: v_mac_f32_e32 v10, v7, v9
+; SDAG-VI-NEXT: v_mac_f32_e32 v11, v6, v8
+; SDAG-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v1, v11 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v2, v4
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v3, v5
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v2, v0
+; SDAG-VI-NEXT: v_or_b32_e32 v1, v3, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v4f32:
; SDAG-CI: ; %bb.0:
@@ -816,39 +863,73 @@ define <4 x half> @v_mad_mix_v4f32(<4 x half> %src0, <4 x half> %src1, <4 x half
; GISEL-GFX906-NEXT: v_mov_b32_e32 v1, v7
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v4f32:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v8, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v10, v4
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v7, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v9, v3
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v11, v5
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v10, v6, v8
+; GISEL-VI-NEXT: v_mac_f32_e32 v11, v7, v9
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v4
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v10
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v3, v5
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v2, v11
+; GISEL-VI-NEXT: v_mov_b32_e32 v4, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v1, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v4f32:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v11, v11
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v10, v10
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v9, v9
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v10, v10
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v11, 16, v5
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_mac_f32_e32 v6, v10, v8
-; GISEL-CI-NEXT: v_mac_f32_e32 v7, v11, v9
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v9, v9
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v11, v11
+; GISEL-CI-NEXT: v_mac_f32_e32 v10, v6, v8
; GISEL-CI-NEXT: v_mac_f32_e32 v4, v0, v2
; GISEL-CI-NEXT: v_mac_f32_e32 v5, v1, v3
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v7
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v6
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v4
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v5
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v4, 16, v0
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v1
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v2, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v1, v3, v4
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v10
+; GISEL-CI-NEXT: v_mac_f32_e32 v11, v7, v9
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v4
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v5
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v11
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_or_b32_e32 v1, v1, v2
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <4 x half> %src0 to <4 x float>
%src1.ext = fpext <4 x half> %src1 to <4 x float>
@@ -895,21 +976,21 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %s
; GFX906-NEXT: v_mov_b32_e32 v0, v3
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_clamp_postcvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mac_f32_e32 v5, v3, v4
-; VI-NEXT: v_mac_f32_e32 v2, v0, v1
-; VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e64 v1, v2 clamp
-; VI-NEXT: v_or_b32_e32 v0, v1, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; SDAG-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v0, v5 clamp dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e64 v1, v2 clamp
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v1, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v2f32_clamp_postcvt:
; SDAG-CI: ; %bb.0:
@@ -952,34 +1033,54 @@ define <2 x half> @v_mad_mix_v2f32_clamp_postcvt(<2 x half> %src0, <2 x half> %s
; GISEL-GFX1100-NEXT: v_mov_b32_e32 v0, v3
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_clamp_postcvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v3, v4
+; GISEL-VI-NEXT: v_mac_f32_e32 v2, v0, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v0, v2 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v1, v5 clamp
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v2f32_clamp_postcvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_mac_f32_e32 v3, v5, v4
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v3
-; GISEL-CI-NEXT: v_mac_f32_e32 v2, v0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v2
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v3
+; GISEL-CI-NEXT: v_mac_f32_e32 v2, v3, v4
+; GISEL-CI-NEXT: v_mac_f32_e32 v5, v0, v1
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v2
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v5
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_max_f32_e32 v1, 0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_max_f32_e32 v0, 0, v0
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_min_f32_e32 v0, 1.0, v0
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
@@ -1040,33 +1141,33 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
; SDAG-GFX906-NEXT: v_mov_b32_e32 v0, v3
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v3f32_clamp_postcvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; VI-NEXT: v_mac_f32_e32 v8, v6, v7
-; VI-NEXT: v_mac_f32_e32 v4, v0, v2
-; VI-NEXT: v_mac_f32_e32 v5, v1, v3
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v8
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v4
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v5
-; VI-NEXT: v_max_f16_e32 v0, 0, v0
-; VI-NEXT: v_max_f16_e32 v3, 0, v1
-; VI-NEXT: v_max_f16_e32 v1, 0, v2
-; VI-NEXT: v_mov_b32_e32 v2, 0x3c00
-; VI-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_min_f16_e32 v2, 1.0, v3
-; VI-NEXT: v_min_f16_e32 v1, 1.0, v1
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v3f32_clamp_postcvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-VI-NEXT: v_mac_f32_e32 v8, v6, v7
+; SDAG-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v8
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v4
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v2, v5
+; SDAG-VI-NEXT: v_max_f16_e32 v0, 0, v0
+; SDAG-VI-NEXT: v_max_f16_e32 v3, 0, v1
+; SDAG-VI-NEXT: v_max_f16_e32 v1, 0, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x3c00
+; SDAG-VI-NEXT: v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; SDAG-VI-NEXT: v_min_f16_e32 v2, 1.0, v3
+; SDAG-VI-NEXT: v_min_f16_e32 v1, 1.0, v1
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v2, v0
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v3f32_clamp_postcvt:
; SDAG-CI: ; %bb.0:
@@ -1148,47 +1249,74 @@ define <3 x half> @v_mad_mix_v3f32_clamp_postcvt(<3 x half> %src0, <3 x half> %s
; GISEL-GFX906-NEXT: v_mov_b32_e32 v0, v3
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v3f32_clamp_postcvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v7, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v8, v4
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GISEL-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v8, v6, v7
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v0, v4 clamp
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v1, v8 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v2, v5 clamp
+; GISEL-VI-NEXT: v_mov_b32_e32 v3, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v3f32_clamp_postcvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_mac_f32_e32 v6, v8, v7
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v6, v6
+; GISEL-CI-NEXT: v_mac_f32_e32 v8, v6, v7
; GISEL-CI-NEXT: v_mac_f32_e32 v4, v0, v2
-; GISEL-CI-NEXT: v_mac_f32_e32 v5, v1, v3
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v4
+; GISEL-CI-NEXT: v_mac_f32_e32 v5, v1, v3
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v8
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v6
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_max_f32_e32 v1, 0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_max_f32_e32 v0, 0, v0
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-CI-NEXT: v_max_f32_e32 v1, 0, v1
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_max_f32_e32 v2, 0, v2
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v1
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v1
; GISEL-CI-NEXT: v_min_f32_e32 v0, 1.0, v0
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v2
+; GISEL-CI-NEXT: v_min_f32_e32 v1, 1.0, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-CI-NEXT: v_min_f32_e32 v2, 1.0, v2
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_bfe_u32 v1, v2, 0, 16
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <3 x half> %src0 to <3 x float>
%src1.ext = fpext <3 x half> %src1 to <3 x float>
@@ -1247,41 +1375,41 @@ define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %s
; GFX906-NEXT: v_mov_b32_e32 v1, v2
; GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v4f32_clamp_postcvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_cvt_f32_f16_sdwa v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; VI-NEXT: v_mac_f32_e32 v10, v7, v9
-; VI-NEXT: v_mac_f32_e32 v11, v6, v8
-; VI-NEXT: v_mac_f32_e32 v4, v0, v2
-; VI-NEXT: v_mac_f32_e32 v5, v1, v3
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v10
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v11
-; VI-NEXT: v_cvt_f16_f32_e32 v2, v4
-; VI-NEXT: v_cvt_f16_f32_e32 v3, v5
-; VI-NEXT: v_max_f16_e32 v0, 0, v0
-; VI-NEXT: v_max_f16_e32 v1, 0, v1
-; VI-NEXT: v_max_f16_e32 v2, 0, v2
-; VI-NEXT: v_max_f16_e32 v3, 0, v3
-; VI-NEXT: v_mov_b32_e32 v4, 0x3c00
-; VI-NEXT: v_min_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_min_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_min_f16_e32 v3, 1.0, v3
-; VI-NEXT: v_min_f16_e32 v2, 1.0, v2
-; VI-NEXT: v_or_b32_e32 v0, v2, v0
-; VI-NEXT: v_or_b32_e32 v1, v3, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v4f32_clamp_postcvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v10, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v11, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-VI-NEXT: v_mac_f32_e32 v10, v7, v9
+; SDAG-VI-NEXT: v_mac_f32_e32 v11, v6, v8
+; SDAG-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; SDAG-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v10
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v11
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v2, v4
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v3, v5
+; SDAG-VI-NEXT: v_max_f16_e32 v0, 0, v0
+; SDAG-VI-NEXT: v_max_f16_e32 v1, 0, v1
+; SDAG-VI-NEXT: v_max_f16_e32 v2, 0, v2
+; SDAG-VI-NEXT: v_max_f16_e32 v3, 0, v3
+; SDAG-VI-NEXT: v_mov_b32_e32 v4, 0x3c00
+; SDAG-VI-NEXT: v_min_f16_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; SDAG-VI-NEXT: v_min_f16_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; SDAG-VI-NEXT: v_min_f16_e32 v3, 1.0, v3
+; SDAG-VI-NEXT: v_min_f16_e32 v2, 1.0, v2
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v2, v0
+; SDAG-VI-NEXT: v_or_b32_e32 v1, v3, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v4f32_clamp_postcvt:
; SDAG-CI: ; %bb.0:
@@ -1354,41 +1482,71 @@ define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %s
; GISEL-GFX1100-NEXT: v_dual_mov_b32 v0, v6 :: v_dual_mov_b32 v1, v7
; GISEL-GFX1100-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v4f32_clamp_postcvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v8, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v10, v4
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v7, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v9, v3
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v11, v5
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mac_f32_e32 v4, v0, v2
+; GISEL-VI-NEXT: v_mac_f32_e32 v10, v6, v8
+; GISEL-VI-NEXT: v_mac_f32_e32 v11, v7, v9
+; GISEL-VI-NEXT: v_mac_f32_e32 v5, v1, v3
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v1, v4 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v0, v10 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v3, v5 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e64 v2, v11 clamp
+; GISEL-VI-NEXT: v_mov_b32_e32 v4, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v1, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v4f32_clamp_postcvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v5
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v11, v11
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v10, v10
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v9, v9
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v10, v10
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v11, 16, v5
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GISEL-CI-NEXT: v_mac_f32_e32 v6, v10, v8
-; GISEL-CI-NEXT: v_mac_f32_e32 v7, v11, v9
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v6, v6
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v7, v7
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v9, v9
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v11, v11
+; GISEL-CI-NEXT: v_mac_f32_e32 v10, v6, v8
; GISEL-CI-NEXT: v_mac_f32_e32 v4, v0, v2
; GISEL-CI-NEXT: v_mac_f32_e32 v5, v1, v3
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v4
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v6
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v7
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v10
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v4
+; GISEL-CI-NEXT: v_mac_f32_e32 v11, v7, v9
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v5
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v11
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; GISEL-CI-NEXT: v_max_f32_e32 v0, 0, v0
; GISEL-CI-NEXT: v_max_f32_e32 v1, 0, v1
+; GISEL-CI-NEXT: v_max_f32_e32 v0, 0, v0
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-CI-NEXT: v_max_f32_e32 v2, 0, v2
@@ -1403,14 +1561,18 @@ define <4 x half> @v_mad_mix_v4f32_clamp_postcvt(<4 x half> %src0, <4 x half> %s
; GISEL-CI-NEXT: v_min_f32_e32 v0, 1.0, v0
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GISEL-CI-NEXT: v_min_f32_e32 v3, 1.0, v3
; GISEL-CI-NEXT: v_min_f32_e32 v2, 1.0, v2
+; GISEL-CI-NEXT: v_min_f32_e32 v3, 1.0, v3
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v2, v0
-; GISEL-CI-NEXT: v_or_b32_e32 v1, v3, v1
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-CI-NEXT: v_bfe_u32 v1, v2, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_or_b32_e32 v1, v1, v2
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <4 x half> %src0 to <4 x float>
%src1.ext = fpext <4 x half> %src1 to <4 x float>
@@ -1716,21 +1878,21 @@ define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %sr
; SDAG-GFX906-NEXT: v_pack_b32_f16 v0, v0, v1
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v2f32_clamp_precvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_mad_f32 v3, v3, v4, v5 clamp
-; VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
-; VI-NEXT: v_cvt_f16_f32_sdwa v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_or_b32_e32 v0, v0, v1
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v2f32_clamp_precvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_mad_f32 v3, v3, v4, v5 clamp
+; SDAG-VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v0, v1
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v2f32_clamp_precvt:
; SDAG-CI: ; %bb.0:
@@ -1784,22 +1946,42 @@ define <2 x half> @v_mad_mix_v2f32_clamp_precvt(<2 x half> %src0, <2 x half> %sr
; GISEL-GFX906-NEXT: v_pack_b32_f16 v0, v1, v0
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v2f32_clamp_precvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v4, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mad_f32 v3, v3, v4, v5 clamp
+; GISEL-VI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v3
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v2f32_clamp_precvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GISEL-CI-NEXT: v_mad_f32 v3, v5, v4, v3 clamp
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
; GISEL-CI-NEXT: v_mad_f32 v0, v0, v1, v2 clamp
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v3
+; GISEL-CI-NEXT: v_mad_f32 v1, v3, v4, v5 clamp
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
@@ -1867,26 +2049,26 @@ define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %sr
; SDAG-GFX906-NEXT: v_pack_b32_f16 v0, v0, v2
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v3f32_clamp_precvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; VI-NEXT: v_mad_f32 v6, v6, v7, v8 clamp
-; VI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
-; VI-NEXT: v_cvt_f16_f32_sdwa v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v3f32_clamp_precvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-VI-NEXT: v_mad_f32 v6, v6, v7, v8 clamp
+; SDAG-VI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v2, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SDAG-VI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v0, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v3f32_clamp_precvt:
; SDAG-CI: ; %bb.0:
@@ -1956,29 +2138,56 @@ define <3 x half> @v_mad_mix_v3f32_clamp_precvt(<3 x half> %src0, <3 x half> %sr
; GISEL-GFX906-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v3f32_clamp_precvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v7, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v8, v4
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; GISEL-VI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
+; GISEL-VI-NEXT: v_mad_f32 v6, v6, v7, v8 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-VI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v2, v6
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-VI-NEXT: v_mov_b32_e32 v3, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v3f32_clamp_precvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_mad_f32 v6, v8, v7, v6 clamp
; GISEL-CI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v6
+; GISEL-CI-NEXT: v_mad_f32 v2, v6, v7, v8 clamp
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v2
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; GISEL-CI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-CI-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <3 x half> %src0 to <3 x float>
%src1.ext = fpext <3 x half> %src1 to <3 x float>
@@ -2054,32 +2263,32 @@ define <4 x half> @v_mad_mix_v4f32_clamp_precvt(<4 x half> %src0, <4 x half> %sr
; SDAG-GFX906-NEXT: v_pack_b32_f16 v1, v1, v2
; SDAG-GFX906-NEXT: s_setpc_b64 s[30:31]
;
-; VI-LABEL: v_mad_mix_v4f32_clamp_precvt:
-; VI: ; %bb.0:
-; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; VI-NEXT: v_cvt_f32_f16_e32 v1, v1
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v2
-; VI-NEXT: v_cvt_f32_f16_e32 v3, v3
-; VI-NEXT: v_cvt_f32_f16_sdwa v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; VI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; VI-NEXT: v_mad_f32 v7, v7, v9, v10 clamp
-; VI-NEXT: v_mad_f32 v6, v6, v8, v11 clamp
-; VI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
-; VI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
-; VI-NEXT: v_cvt_f16_f32_sdwa v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_sdwa v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; VI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; VI-NEXT: v_or_b32_e32 v0, v0, v3
-; VI-NEXT: v_or_b32_e32 v1, v1, v2
-; VI-NEXT: s_setpc_b64 s[30:31]
+; SDAG-VI-LABEL: v_mad_mix_v4f32_clamp_precvt:
+; SDAG-VI: ; %bb.0:
+; SDAG-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v6, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v7, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v9, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v10, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_sdwa v11, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v5, v5
+; SDAG-VI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; SDAG-VI-NEXT: v_mad_f32 v7, v7, v9, v10 clamp
+; SDAG-VI-NEXT: v_mad_f32 v6, v6, v8, v11 clamp
+; SDAG-VI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
+; SDAG-VI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v2, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_sdwa v3, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; SDAG-VI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; SDAG-VI-NEXT: v_or_b32_e32 v0, v0, v3
+; SDAG-VI-NEXT: v_or_b32_e32 v1, v1, v2
+; SDAG-VI-NEXT: s_setpc_b64 s[30:31]
;
; SDAG-CI-LABEL: v_mad_mix_v4f32_clamp_precvt:
; SDAG-CI: ; %bb.0:
@@ -2164,38 +2373,72 @@ define <4 x half> @v_mad_mix_v4f32_clamp_precvt(<4 x half> %src0, <4 x half> %sr
; GISEL-GFX906-NEXT: v_pack_b32_f16 v1, v2, v1
; GISEL-GFX906-NEXT: s_setpc_b64 s[30:31]
;
+; GISEL-VI-LABEL: v_mad_mix_v4f32_clamp_precvt:
+; GISEL-VI: ; %bb.0:
+; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v6, v0
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v7, v1
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v8, v2
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v9, v3
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v3, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v10, v4
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v4, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_cvt_f32_f16_e32 v11, v5
+; GISEL-VI-NEXT: v_cvt_f32_f16_sdwa v5, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GISEL-VI-NEXT: v_mad_f32 v6, v6, v8, v10 clamp
+; GISEL-VI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
+; GISEL-VI-NEXT: v_mad_f32 v2, v7, v9, v11 clamp
+; GISEL-VI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v3, v6
+; GISEL-VI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-VI-NEXT: v_mov_b32_e32 v4, 16
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GISEL-VI-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
+;
; GISEL-CI-LABEL: v_mad_mix_v4f32_clamp_precvt:
; GISEL-CI: ; %bb.0:
; GISEL-CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GISEL-CI-NEXT: v_lshrrev_b32_e32 v11, 16, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v11, v11
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v10, v10
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v9, v9
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v8, 16, v2
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v0, v0
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v6, v6
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v8, v8
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v10, v10
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; GISEL-CI-NEXT: v_lshrrev_b32_e32 v11, 16, v5
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v7, v7
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v3, v3
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v9, v9
; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v5, v5
-; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v4, v4
-; GISEL-CI-NEXT: v_mad_f32 v6, v10, v8, v6 clamp
-; GISEL-CI-NEXT: v_mad_f32 v7, v11, v9, v7 clamp
-; GISEL-CI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
+; GISEL-CI-NEXT: v_cvt_f32_f16_e32 v11, v11
; GISEL-CI-NEXT: v_mad_f32 v0, v0, v2, v4 clamp
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v6
-; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v7
+; GISEL-CI-NEXT: v_mad_f32 v2, v6, v8, v10 clamp
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GISEL-CI-NEXT: v_mad_f32 v1, v1, v3, v5 clamp
+; GISEL-CI-NEXT: v_mad_f32 v3, v7, v9, v11 clamp
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v3, v3
; GISEL-CI-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GISEL-CI-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v2
+; GISEL-CI-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GISEL-CI-NEXT: v_bfe_u32 v1, v1, 0, 16
; GISEL-CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GISEL-CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GISEL-CI-NEXT: v_or_b32_e32 v0, v0, v3
; GISEL-CI-NEXT: v_or_b32_e32 v1, v1, v2
; GISEL-CI-NEXT: s_setpc_b64 s[30:31]
%src0.ext = fpext <4 x half> %src0 to <4 x float>
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 1d0565d150ac3..14f56571ca056 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -1,1097 +1,1886 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-SDAG %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-GISEL %s
+; RUN: llc -global-isel -mtriple=amdgpu9.42-amd-amdhsa %s -o - | FileCheck -check-prefixes=GFX942,GFX942-GISEL %s
define void @memset_p0_varsize_align_4_varsetval(ptr addrspace(0) align 4 %dst, i8 %setval, i64 %size) #0 {
-; GFX942-LABEL: memset_p0_varsize_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v10
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB0_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s4
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB0_2
-; GFX942-NEXT: .LBB0_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB0_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v10
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: flat_store_byte v[4:5], v2
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB0_5
-; GFX942-NEXT: .LBB0_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p0_varsize_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_2
+; GFX942-SDAG-NEXT: .LBB0_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB0_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: flat_store_byte v[4:5], v2
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB0_5
+; GFX942-SDAG-NEXT: .LBB0_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p0_varsize_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v12, vcc, v10, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v13, vcc, 0, v11, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB0_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB0_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, v0, v14
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v15, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[14:15], v[4:7]
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB0_2
+; GFX942-GISEL-NEXT: .LBB0_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB0_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 4, v[10:11]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB0_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: flat_store_byte v[4:5], v2
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB0_5
+; GFX942-GISEL-NEXT: .LBB0_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p0.i64(ptr addrspace(0) noundef nonnull align 4 %dst, i8 %setval, i64 %size, i1 false)
ret void
}
define void @memset_p1_varsize_align_4_varsetval(ptr addrspace(1) align 4 %dst, i8 %setval, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsize_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v10, v3
-; GFX942-NEXT: v_and_b32_e32 v12, -16, v10
-; GFX942-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v10
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB1_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s4
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB1_2
-; GFX942-NEXT: .LBB1_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB1_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v10
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[4:5], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB1_5
-; GFX942-NEXT: .LBB1_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsize_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v12, -16, v10
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[14:15], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[12:13]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_2
+; GFX942-SDAG-NEXT: .LBB1_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB1_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v10
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[10:11]
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB1_5
+; GFX942-SDAG-NEXT: .LBB1_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsize_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v3
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v10
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v12, vcc, v10, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v13, vcc, 0, v11, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB1_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB1_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[14:15], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v14, vcc, v0, v14
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v15, vcc, v1, v15, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[12:13]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[14:15], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB1_2
+; GFX942-GISEL-NEXT: .LBB1_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB1_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[4:5], 4, v[10:11]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[4:5], 4, v[0:1]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB1_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB1_5
+; GFX942-GISEL-NEXT: .LBB1_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 %setval, i64 %size, i1 false)
ret void
}
define void @memset_p3_varsize_align_4_varsetval(ptr addrspace(3) align 4 %dst, i8 %setval, i64 %size) #0 {
-; GFX942-LABEL: memset_p3_varsize_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: v_and_b32_e32 v4, -16, v2
-; GFX942-NEXT: v_and_b32_e32 v10, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB2_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-NEXT: v_perm_b32 v6, v1, v1, s4
-; GFX942-NEXT: v_mov_b32_e32 v3, v6
-; GFX942-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-NEXT: v_mov_b32_e32 v9, v0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-NEXT: ds_write2_b32 v9, v8, v7 offset0:2 offset1:3
-; GFX942-NEXT: ds_write2_b32 v9, v6, v3 offset1:1
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_add_u32_e32 v9, 16, v9
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB2_2
-; GFX942-NEXT: .LBB2_3: ; %Flow7
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB2_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_add_u32_e32 v0, v0, v2
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB2_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: ds_write_b8 v0, v1
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_add_u32_e32 v0, 1, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB2_5
-; GFX942-NEXT: .LBB2_6: ; %Flow5
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p3_varsize_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v6, v1, v1, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-SDAG-NEXT: ds_write2_b32 v9, v8, v7 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v9, v6, v3 offset1:1
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v9, 16, v9
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB2_2
+; GFX942-SDAG-NEXT: .LBB2_3: ; %Flow7
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB2_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, v0, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB2_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: ds_write_b8 v0, v1
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB2_5
+; GFX942-SDAG-NEXT: .LBB2_6: ; %Flow5
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p3_varsize_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v6, vcc, v2, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v7, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB2_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v9, v8, 8, v8
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v10, 16, v8
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v8, 24, v8
+; GFX942-GISEL-NEXT: v_or3_b32 v8, v9, v10, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v10, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v11, v8
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v12, v0
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB2_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-GISEL-NEXT: ds_write2_b64 v12, v[8:9], v[10:11] offset1:1
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v12, 16, v12
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB2_2
+; GFX942-GISEL-NEXT: .LBB2_3: ; %Flow7
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB2_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 4, v[2:3]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, v0, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB2_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-GISEL-NEXT: ds_write_b8 v0, v1
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB2_5
+; GFX942-GISEL-NEXT: .LBB2_6: ; %Flow5
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p3.i64(ptr addrspace(3) noundef nonnull align 4 %dst, i8 %setval, i64 %size, i1 false)
ret void
}
define void @memset_p5_varsize_align_4_varsetval(ptr addrspace(5) align 4 %dst, i8 %setval, i64 %size) #0 {
-; GFX942-LABEL: memset_p5_varsize_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v5, v3
-; GFX942-NEXT: v_and_b32_e32 v4, -16, v2
-; GFX942-NEXT: v_and_b32_e32 v10, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB3_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: s_mov_b32 s4, 0x4040404
-; GFX942-NEXT: v_perm_b32 v6, v1, v1, s4
-; GFX942-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-NEXT: v_mov_b32_e32 v9, v6
-; GFX942-NEXT: v_mov_b32_e32 v3, v0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB3_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-NEXT: scratch_store_dwordx4 v3, v[6:9], off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_add_u32_e32 v3, 16, v3
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB3_2
-; GFX942-NEXT: .LBB3_3: ; %Flow7
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB3_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_add_u32_e32 v0, v0, v2
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB3_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: scratch_store_byte v0, v1, off
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: v_add_u32_e32 v0, 1, v0
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB3_5
-; GFX942-NEXT: .LBB3_6: ; %Flow5
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p5_varsize_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, -16, v2
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB3_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: s_mov_b32 s4, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v6, v1, v1, s4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB3_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v3, v[6:9], off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v3, 16, v3
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB3_2
+; GFX942-SDAG-NEXT: .LBB3_3: ; %Flow7
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB3_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, v0, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB3_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: scratch_store_byte v0, v1, off
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB3_5
+; GFX942-SDAG-NEXT: .LBB3_6: ; %Flow5
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p5_varsize_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB3_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v5, v4, 8, v4
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v6, 16, v4
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 24, v4
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v5, v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v12, v0
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB3_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 16
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v12, v[4:7], off
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v12, 16, v12
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB3_2
+; GFX942-GISEL-NEXT: .LBB3_3: ; %Flow7
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB3_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 4, v[2:3]
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 4, v2
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, v0, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB3_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s4, s4, 1
+; GFX942-GISEL-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off
+; GFX942-GISEL-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB3_5
+; GFX942-GISEL-NEXT: .LBB3_6: ; %Flow5
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p5.i64(ptr addrspace(5) noundef nonnull align 4 %dst, i8 %setval, i64 %size, i1 false)
ret void
}
define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p0_sz1055_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_mov_b32_e32 v8, v4
-; GFX942-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-NEXT: v_mov_b32_e32 v10, v4
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_mov_b32_e32 v12, v4
-; GFX942-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-NEXT: v_mov_b32_e32 v14, v4
-; GFX942-NEXT: v_mov_b32_e32 v15, v4
-; GFX942-NEXT: v_mov_b32_e32 v16, v4
-; GFX942-NEXT: v_mov_b32_e32 v17, v4
-; GFX942-NEXT: v_mov_b32_e32 v18, v4
-; GFX942-NEXT: v_mov_b32_e32 v19, v4
-; GFX942-NEXT: v_mov_b32_e32 v20, v4
-; GFX942-NEXT: v_mov_b32_e32 v21, v4
-; GFX942-NEXT: v_mov_b32_e32 v22, v4
-; GFX942-NEXT: v_mov_b32_e32 v23, v4
-; GFX942-NEXT: v_mov_b32_e32 v24, v4
-; GFX942-NEXT: v_mov_b32_e32 v25, v4
-; GFX942-NEXT: v_mov_b32_e32 v26, v4
-; GFX942-NEXT: v_mov_b32_e32 v27, v4
-; GFX942-NEXT: v_mov_b32_e32 v28, v4
-; GFX942-NEXT: v_mov_b32_e32 v29, v4
-; GFX942-NEXT: v_mov_b32_e32 v30, v4
-; GFX942-NEXT: v_mov_b32_e32 v31, v4
-; GFX942-NEXT: v_mov_b32_e32 v32, v4
-; GFX942-NEXT: v_mov_b32_e32 v33, v4
-; GFX942-NEXT: v_mov_b32_e32 v34, v4
-; GFX942-NEXT: v_mov_b32_e32 v35, v4
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x70
-; GFX942-NEXT: s_mov_b64 s[4:5], 0x60
-; GFX942-NEXT: s_mov_b64 s[6:7], 0x50
-; GFX942-NEXT: s_mov_b64 s[8:9], 0xf0
-; GFX942-NEXT: s_mov_b64 s[10:11], 0xe0
-; GFX942-NEXT: s_mov_b64 s[12:13], 0xd0
-; GFX942-NEXT: s_mov_b64 s[14:15], 0xc0
-; GFX942-NEXT: s_mov_b64 s[16:17], 0xb0
-; GFX942-NEXT: s_mov_b64 s[18:19], 0xa0
-; GFX942-NEXT: s_mov_b64 s[20:21], 0x90
-; GFX942-NEXT: v_mov_b64_e32 v[36:37], 0x400
-; GFX942-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
-; GFX942-NEXT: v_lshl_add_u64 v[48:49], v[38:39], 0, s[2:3]
-; GFX942-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-NEXT: v_lshl_add_u64 v[50:51], v[38:39], 0, s[4:5]
-; GFX942-NEXT: v_lshl_add_u64 v[52:53], v[38:39], 0, s[6:7]
-; GFX942-NEXT: flat_store_dwordx4 v[38:39], v[20:23] offset:64
-; GFX942-NEXT: v_lshl_add_u64 v[54:55], v[38:39], 0, 48
-; GFX942-NEXT: flat_store_dwordx4 v[38:39], v[12:15] offset:32
-; GFX942-NEXT: flat_store_dwordx4 v[38:39], v[8:11] offset:16
-; GFX942-NEXT: flat_store_dwordx4 v[38:39], v[4:7]
-; GFX942-NEXT: v_lshl_add_u64 v[40:41], v[38:39], 0, s[8:9]
-; GFX942-NEXT: v_lshl_add_u64 v[42:43], v[38:39], 0, s[10:11]
-; GFX942-NEXT: v_lshl_add_u64 v[44:45], v[38:39], 0, s[12:13]
-; GFX942-NEXT: v_lshl_add_u64 v[46:47], v[38:39], 0, s[14:15]
-; GFX942-NEXT: v_lshl_add_u64 v[56:57], v[38:39], 0, s[16:17]
-; GFX942-NEXT: v_lshl_add_u64 v[58:59], v[38:39], 0, s[18:19]
-; GFX942-NEXT: v_lshl_add_u64 v[60:61], v[38:39], 0, s[20:21]
-; GFX942-NEXT: flat_store_dwordx4 v[38:39], v[4:7] offset:128
-; GFX942-NEXT: flat_store_dwordx4 v[48:49], v[32:35]
-; GFX942-NEXT: flat_store_dwordx4 v[50:51], v[28:31]
-; GFX942-NEXT: flat_store_dwordx4 v[52:53], v[24:27]
-; GFX942-NEXT: flat_store_dwordx4 v[54:55], v[16:19]
-; GFX942-NEXT: flat_store_dwordx4 v[40:41], v[32:35]
-; GFX942-NEXT: flat_store_dwordx4 v[42:43], v[28:31]
-; GFX942-NEXT: flat_store_dwordx4 v[44:45], v[24:27]
-; GFX942-NEXT: flat_store_dwordx4 v[46:47], v[20:23]
-; GFX942-NEXT: flat_store_dwordx4 v[56:57], v[16:19]
-; GFX942-NEXT: flat_store_dwordx4 v[58:59], v[12:15]
-; GFX942-NEXT: flat_store_dwordx4 v[60:61], v[8:11]
-; GFX942-NEXT: s_cbranch_vccnz .LBB4_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s0
-; GFX942-NEXT: v_lshlrev_b16_e32 v3, 8, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:1024
-; GFX942-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
-; GFX942-NEXT: flat_store_short v[0:1], v3 offset:1052
-; GFX942-NEXT: flat_store_byte v[0:1], v2 offset:1054
-; GFX942-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p0_sz1055_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a12, v60 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a13, v61 ; Reload Reuse
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0xf0
+; GFX942-SDAG-NEXT: s_mov_b64 s[10:11], 0xe0
+; GFX942-SDAG-NEXT: s_mov_b64 s[12:13], 0xd0
+; GFX942-SDAG-NEXT: s_mov_b64 s[14:15], 0xc0
+; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
+; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
+; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
+; GFX942-SDAG-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[38:39], 0, s[2:3]
+; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[38:39], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[38:39], 0, s[6:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[20:23] offset:64
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[38:39], 0, 48
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[12:15] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[8:11] offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[38:39], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[38:39], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[38:39], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[38:39], 0, s[14:15]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[38:39], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[38:39], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[60:61], v[38:39], 0, s[20:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[4:7] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[32:35]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[28:31]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[24:27]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[16:19]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[32:35]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[28:31]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[24:27]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[20:23]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[16:19]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[12:15]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[60:61], v[8:11]
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB4_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:1024
+; GFX942-SDAG-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
+; GFX942-SDAG-NEXT: flat_store_short v[0:1], v3 offset:1052
+; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 offset:1054
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p0_sz1055_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0x400
+; GFX942-GISEL-NEXT: .LBB4_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, v0, v10
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v11, vcc
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:16
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:32
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:48
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:64
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:80
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:96
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:112
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:128
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:144
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:160
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:176
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:192
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:208
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:224
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[10:11], v[4:7] offset:240
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB4_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v6, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
+; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[0:1], v[6:9] offset:1024
+; GFX942-GISEL-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
+; GFX942-GISEL-NEXT: flat_store_short v[0:1], v3 offset:1052
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1054
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p0.i64(ptr addrspace(0) noundef nonnull align 4 %dst, i8 %setval, i64 1055, i1 false)
ret void
}
define void @memset_p0_sz2048_align_4_varsetval(ptr addrspace(0) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p0_sz2048_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: s_mov_b64 s[2:3], 0x70
-; GFX942-NEXT: s_mov_b64 s[4:5], 0x60
-; GFX942-NEXT: s_mov_b64 s[6:7], 0x50
-; GFX942-NEXT: s_mov_b64 s[8:9], 0xf0
-; GFX942-NEXT: s_mov_b64 s[10:11], 0xe0
-; GFX942-NEXT: s_mov_b64 s[12:13], 0xd0
-; GFX942-NEXT: s_mov_b64 s[14:15], 0xc0
-; GFX942-NEXT: s_mov_b64 s[16:17], 0xb0
-; GFX942-NEXT: s_mov_b64 s[18:19], 0xa0
-; GFX942-NEXT: s_mov_b64 s[20:21], 0x90
-; GFX942-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-NEXT: v_lshl_add_u64 v[38:39], v[36:37], 0, s[2:3]
-; GFX942-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-NEXT: v_lshl_add_u64 v[48:49], v[36:37], 0, s[4:5]
-; GFX942-NEXT: v_lshl_add_u64 v[50:51], v[36:37], 0, s[6:7]
-; GFX942-NEXT: flat_store_dwordx4 v[36:37], v[18:21] offset:64
-; GFX942-NEXT: v_lshl_add_u64 v[52:53], v[36:37], 0, 48
-; GFX942-NEXT: flat_store_dwordx4 v[36:37], v[10:13] offset:32
-; GFX942-NEXT: flat_store_dwordx4 v[36:37], v[6:9] offset:16
-; GFX942-NEXT: flat_store_dwordx4 v[36:37], v[2:5]
-; GFX942-NEXT: v_lshl_add_u64 v[54:55], v[36:37], 0, s[8:9]
-; GFX942-NEXT: v_lshl_add_u64 v[40:41], v[36:37], 0, s[10:11]
-; GFX942-NEXT: v_lshl_add_u64 v[42:43], v[36:37], 0, s[12:13]
-; GFX942-NEXT: v_lshl_add_u64 v[44:45], v[36:37], 0, s[14:15]
-; GFX942-NEXT: v_lshl_add_u64 v[46:47], v[36:37], 0, s[16:17]
-; GFX942-NEXT: v_lshl_add_u64 v[56:57], v[36:37], 0, s[18:19]
-; GFX942-NEXT: v_lshl_add_u64 v[58:59], v[36:37], 0, s[20:21]
-; GFX942-NEXT: flat_store_dwordx4 v[36:37], v[2:5] offset:128
-; GFX942-NEXT: flat_store_dwordx4 v[38:39], v[30:33]
-; GFX942-NEXT: flat_store_dwordx4 v[48:49], v[26:29]
-; GFX942-NEXT: flat_store_dwordx4 v[50:51], v[22:25]
-; GFX942-NEXT: flat_store_dwordx4 v[52:53], v[14:17]
-; GFX942-NEXT: flat_store_dwordx4 v[54:55], v[30:33]
-; GFX942-NEXT: flat_store_dwordx4 v[40:41], v[26:29]
-; GFX942-NEXT: flat_store_dwordx4 v[42:43], v[22:25]
-; GFX942-NEXT: flat_store_dwordx4 v[44:45], v[18:21]
-; GFX942-NEXT: flat_store_dwordx4 v[46:47], v[14:17]
-; GFX942-NEXT: flat_store_dwordx4 v[56:57], v[10:13]
-; GFX942-NEXT: flat_store_dwordx4 v[58:59], v[6:9]
-; GFX942-NEXT: s_cbranch_vccnz .LBB5_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
-; GFX942-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
-; GFX942-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p0_sz2048_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a0, v40 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a1, v41 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a2, v42 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a3, v43 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a4, v44 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a5, v45 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a6, v46 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a7, v47 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a8, v56 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a9, v57 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a10, v58 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_write_b32 a11, v59 ; Reload Reuse
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[2:3], 0x70
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0x60
+; GFX942-SDAG-NEXT: s_mov_b64 s[6:7], 0x50
+; GFX942-SDAG-NEXT: s_mov_b64 s[8:9], 0xf0
+; GFX942-SDAG-NEXT: s_mov_b64 s[10:11], 0xe0
+; GFX942-SDAG-NEXT: s_mov_b64 s[12:13], 0xd0
+; GFX942-SDAG-NEXT: s_mov_b64 s[14:15], 0xc0
+; GFX942-SDAG-NEXT: s_mov_b64 s[16:17], 0xb0
+; GFX942-SDAG-NEXT: s_mov_b64 s[18:19], 0xa0
+; GFX942-SDAG-NEXT: s_mov_b64 s[20:21], 0x90
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[36:37], 0, s[2:3]
+; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[48:49], v[36:37], 0, s[4:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[50:51], v[36:37], 0, s[6:7]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[18:21] offset:64
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[52:53], v[36:37], 0, 48
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[10:13] offset:32
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[6:9] offset:16
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[54:55], v[36:37], 0, s[8:9]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[40:41], v[36:37], 0, s[10:11]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[42:43], v[36:37], 0, s[12:13]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[44:45], v[36:37], 0, s[14:15]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[46:47], v[36:37], 0, s[16:17]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[56:57], v[36:37], 0, s[18:19]
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[58:59], v[36:37], 0, s[20:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[36:37], v[2:5] offset:128
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[38:39], v[30:33]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[48:49], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[50:51], v[22:25]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[52:53], v[14:17]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[54:55], v[30:33]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[40:41], v[26:29]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[42:43], v[22:25]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[44:45], v[18:21]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[46:47], v[14:17]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[56:57], v[10:13]
+; GFX942-SDAG-NEXT: flat_store_dwordx4 v[58:59], v[6:9]
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB5_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v58, a10 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v57, a9 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v56, a8 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v47, a7 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v46, a6 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v45, a5 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v44, a4 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v43, a3 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v42, a2 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v41, a1 ; Reload Reuse
+; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v40, a0 ; Reload Reuse
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p0_sz2048_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v3, v2, 8, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v3, v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x800
+; GFX942-GISEL-NEXT: .LBB5_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, v0, v8
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5]
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:16
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:32
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:48
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:64
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:80
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:96
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:112
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:128
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:144
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:160
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:176
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:192
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:208
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:224
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[8:9], v[2:5] offset:240
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB5_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p0.i64(ptr addrspace(0) noundef nonnull align 4 %dst, i8 %setval, i64 2048, i1 false)
ret void
}
define void @memset_p1_sz1055_align_4_varsetval(ptr addrspace(1) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p1_sz1055_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_mov_b32_e32 v8, v4
-; GFX942-NEXT: v_mov_b32_e32 v9, v4
-; GFX942-NEXT: v_mov_b32_e32 v10, v4
-; GFX942-NEXT: v_mov_b32_e32 v11, v4
-; GFX942-NEXT: v_mov_b32_e32 v12, v4
-; GFX942-NEXT: v_mov_b32_e32 v13, v4
-; GFX942-NEXT: v_mov_b32_e32 v14, v4
-; GFX942-NEXT: v_mov_b32_e32 v15, v4
-; GFX942-NEXT: v_mov_b32_e32 v16, v4
-; GFX942-NEXT: v_mov_b32_e32 v17, v4
-; GFX942-NEXT: v_mov_b32_e32 v18, v4
-; GFX942-NEXT: v_mov_b32_e32 v19, v4
-; GFX942-NEXT: v_mov_b32_e32 v20, v4
-; GFX942-NEXT: v_mov_b32_e32 v21, v4
-; GFX942-NEXT: v_mov_b32_e32 v22, v4
-; GFX942-NEXT: v_mov_b32_e32 v23, v4
-; GFX942-NEXT: v_mov_b32_e32 v24, v4
-; GFX942-NEXT: v_mov_b32_e32 v25, v4
-; GFX942-NEXT: v_mov_b32_e32 v26, v4
-; GFX942-NEXT: v_mov_b32_e32 v27, v4
-; GFX942-NEXT: v_mov_b32_e32 v28, v4
-; GFX942-NEXT: v_mov_b32_e32 v29, v4
-; GFX942-NEXT: v_mov_b32_e32 v30, v4
-; GFX942-NEXT: v_mov_b32_e32 v31, v4
-; GFX942-NEXT: v_mov_b32_e32 v32, v4
-; GFX942-NEXT: v_mov_b32_e32 v33, v4
-; GFX942-NEXT: v_mov_b32_e32 v34, v4
-; GFX942-NEXT: v_mov_b32_e32 v35, v4
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_mov_b64_e32 v[36:37], 0x400
-; GFX942-NEXT: .LBB6_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
-; GFX942-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:112
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:96
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:80
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:64
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:48
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:32
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:16
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[4:7], off
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:240
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:224
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:208
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:192
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:176
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:160
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:144
-; GFX942-NEXT: global_store_dwordx4 v[38:39], v[4:7], off offset:128
-; GFX942-NEXT: s_cbranch_vccnz .LBB6_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v4, v2, v2, s0
-; GFX942-NEXT: v_lshlrev_b16_e32 v3, 8, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-NEXT: global_store_dwordx4 v[0:1], v[4:7], off offset:1024
-; GFX942-NEXT: global_store_dwordx3 v[0:1], v[4:6], off offset:1040
-; GFX942-NEXT: global_store_short v[0:1], v3, off offset:1052
-; GFX942-NEXT: global_store_byte v[0:1], v2, off offset:1054
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_sz1055_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v34, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v35, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[36:37], 0x400
+; GFX942-SDAG-NEXT: .LBB6_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[38:39], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[36:37]
+; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:112
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:96
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:80
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:64
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[4:7], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[32:35], off offset:240
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[28:31], off offset:224
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[24:27], off offset:208
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[20:23], off offset:192
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[16:19], off offset:176
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[12:15], off offset:160
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[8:11], off offset:144
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[38:39], v[4:7], off offset:128
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB6_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
+; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[0:1], v[4:7], off offset:1024
+; GFX942-SDAG-NEXT: global_store_dwordx3 v[0:1], v[4:6], off offset:1040
+; GFX942-SDAG-NEXT: global_store_short v[0:1], v3, off offset:1052
+; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off offset:1054
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_sz1055_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 0x400
+; GFX942-GISEL-NEXT: .LBB6_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v10, vcc, v0, v10
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v11, vcc, v1, v11, vcc
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:112
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:128
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:144
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:160
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:176
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:192
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:208
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:224
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[10:11], v[4:7], off offset:240
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB6_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX942-GISEL-NEXT: v_or3_b32 v6, v4, v5, v3
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
+; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:1024
+; GFX942-GISEL-NEXT: global_store_dwordx3 v[0:1], v[4:6], off offset:1040
+; GFX942-GISEL-NEXT: global_store_short v[0:1], v3, off offset:1052
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1054
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 %setval, i64 1055, i1 false)
ret void
}
define void @memset_p1_sz2048_align_4_varsetval(ptr addrspace(1) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p1_sz2048_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v2, v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-NEXT: .LBB7_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-NEXT: s_and_b64 vcc, exec, vcc
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:112
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:96
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:80
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:64
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:48
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:32
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:16
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[2:5], off
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:240
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:224
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:208
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:192
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:176
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:160
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:144
-; GFX942-NEXT: global_store_dwordx4 v[36:37], v[2:5], off offset:128
-; GFX942-NEXT: s_cbranch_vccnz .LBB7_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_sz2048_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v2, v2, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: .LBB7_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[36:37], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: s_and_b64 vcc, exec, vcc
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:112
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:96
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:80
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:64
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[2:5], off
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[30:33], off offset:240
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[26:29], off offset:224
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[22:25], off offset:208
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[18:21], off offset:192
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[14:17], off offset:176
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[10:13], off offset:160
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[6:9], off offset:144
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[36:37], v[2:5], off offset:128
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB7_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_sz2048_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v3, v2, 8, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v3, v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x800
+; GFX942-GISEL-NEXT: .LBB7_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, v0, v8
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, v1, v9, vcc
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:64
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:80
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:96
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:112
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:128
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:144
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:160
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:176
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:192
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:208
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:224
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[8:9], v[2:5], off offset:240
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB7_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 %setval, i64 2048, i1 false)
ret void
}
define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p3_sz1055_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v1, v1, s0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-NEXT: v_mov_b32_e32 v36, v0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-NEXT: .LBB8_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-NEXT: ds_write2_b32 v36, v32, v33 offset0:30 offset1:31
-; GFX942-NEXT: ds_write2_b32 v36, v30, v31 offset0:28 offset1:29
-; GFX942-NEXT: ds_write2_b32 v36, v28, v29 offset0:26 offset1:27
-; GFX942-NEXT: ds_write2_b32 v36, v26, v27 offset0:24 offset1:25
-; GFX942-NEXT: ds_write2_b32 v36, v24, v25 offset0:22 offset1:23
-; GFX942-NEXT: ds_write2_b32 v36, v22, v23 offset0:20 offset1:21
-; GFX942-NEXT: ds_write2_b32 v36, v20, v21 offset0:18 offset1:19
-; GFX942-NEXT: ds_write2_b32 v36, v18, v19 offset0:16 offset1:17
-; GFX942-NEXT: ds_write2_b32 v36, v16, v17 offset0:14 offset1:15
-; GFX942-NEXT: ds_write2_b32 v36, v14, v15 offset0:12 offset1:13
-; GFX942-NEXT: ds_write2_b32 v36, v12, v13 offset0:10 offset1:11
-; GFX942-NEXT: ds_write2_b32 v36, v10, v11 offset0:8 offset1:9
-; GFX942-NEXT: ds_write2_b32 v36, v8, v9 offset0:6 offset1:7
-; GFX942-NEXT: ds_write2_b32 v36, v6, v7 offset0:4 offset1:5
-; GFX942-NEXT: ds_write2_b32 v36, v4, v5 offset0:2 offset1:3
-; GFX942-NEXT: ds_write2_b32 v36, v2, v3 offset1:1
-; GFX942-NEXT: ds_write2_b32 v36, v32, v33 offset0:62 offset1:63
-; GFX942-NEXT: ds_write2_b32 v36, v30, v31 offset0:60 offset1:61
-; GFX942-NEXT: ds_write2_b32 v36, v28, v29 offset0:58 offset1:59
-; GFX942-NEXT: ds_write2_b32 v36, v26, v27 offset0:56 offset1:57
-; GFX942-NEXT: ds_write2_b32 v36, v24, v25 offset0:54 offset1:55
-; GFX942-NEXT: ds_write2_b32 v36, v22, v23 offset0:52 offset1:53
-; GFX942-NEXT: ds_write2_b32 v36, v20, v21 offset0:50 offset1:51
-; GFX942-NEXT: ds_write2_b32 v36, v18, v19 offset0:48 offset1:49
-; GFX942-NEXT: ds_write2_b32 v36, v16, v17 offset0:46 offset1:47
-; GFX942-NEXT: ds_write2_b32 v36, v14, v15 offset0:44 offset1:45
-; GFX942-NEXT: ds_write2_b32 v36, v12, v13 offset0:42 offset1:43
-; GFX942-NEXT: ds_write2_b32 v36, v10, v11 offset0:40 offset1:41
-; GFX942-NEXT: ds_write2_b32 v36, v8, v9 offset0:38 offset1:39
-; GFX942-NEXT: ds_write2_b32 v36, v6, v7 offset0:36 offset1:37
-; GFX942-NEXT: ds_write2_b32 v36, v4, v5 offset0:34 offset1:35
-; GFX942-NEXT: ds_write2_b32 v36, v2, v3 offset0:32 offset1:33
-; GFX942-NEXT: v_add_u32_e32 v36, 0x100, v36
-; GFX942-NEXT: s_cbranch_vccnz .LBB8_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_add_u32_e32 v2, 0x400, v0
-; GFX942-NEXT: v_add_u32_e32 v3, 0x408, v0
-; GFX942-NEXT: v_perm_b32 v4, v1, v1, s0
-; GFX942-NEXT: ds_write2_b32 v3, v4, v4 offset1:1
-; GFX942-NEXT: ds_write2_b32 v2, v4, v4 offset1:1
-; GFX942-NEXT: v_add_u32_e32 v2, 0x410, v0
-; GFX942-NEXT: ds_write2_b32 v2, v4, v4 offset1:1
-; GFX942-NEXT: ds_write_b32 v0, v4 offset:1048
-; GFX942-NEXT: v_lshlrev_b16_e32 v2, 8, v1
-; GFX942-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-NEXT: ds_write_b16 v0, v2 offset:1052
-; GFX942-NEXT: ds_write_b8 v0, v1 offset:1054
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p3_sz1055_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: .LBB8_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v32, v33 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v30, v31 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v28, v29 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v26, v27 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v24, v25 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v22, v23 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v20, v21 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v18, v19 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v16, v17 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v14, v15 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v12, v13 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v10, v11 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v8, v9 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v6, v7 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v4, v5 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v36, v2, v3 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB8_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 0x400, v0
+; GFX942-SDAG-NEXT: v_add_u32_e32 v3, 0x408, v0
+; GFX942-SDAG-NEXT: v_perm_b32 v4, v1, v1, s0
+; GFX942-SDAG-NEXT: ds_write2_b32 v3, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v2, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 0x410, v0
+; GFX942-SDAG-NEXT: ds_write2_b32 v2, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: ds_write_b32 v0, v4 offset:1048
+; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v2, 8, v1
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: ds_write_b16 v0, v2 offset:1052
+; GFX942-SDAG-NEXT: ds_write_b8 v0, v1 offset:1054
+; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p3_sz1055_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v3, v2, 8, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v3, v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x400
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v0
+; GFX942-GISEL-NEXT: .LBB8_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset1:1
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:2 offset1:3
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:4 offset1:5
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:6 offset1:7
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:8 offset1:9
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:10 offset1:11
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:12 offset1:13
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:14 offset1:15
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:16 offset1:17
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:18 offset1:19
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:20 offset1:21
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:22 offset1:23
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:24 offset1:25
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:26 offset1:27
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:28 offset1:29
+; GFX942-GISEL-NEXT: ds_write2_b64 v8, v[2:3], v[4:5] offset0:30 offset1:31
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v8, 0x100, v8
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB8_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v3, v2, 8, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v3, v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:128 offset1:129
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
+; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
+; GFX942-GISEL-NEXT: v_add_u32_e32 v5, 0x410, v0
+; GFX942-GISEL-NEXT: ds_write2_b32 v5, v4, v4 offset1:1
+; GFX942-GISEL-NEXT: ds_write_b32 v0, v2 offset:1048
+; GFX942-GISEL-NEXT: ds_write_b16 v0, v3 offset:1052
+; GFX942-GISEL-NEXT: ds_write_b8 v0, v1 offset:1054
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p3.i64(ptr addrspace(3) noundef nonnull align 4 %dst, i8 %setval, i64 1055, i1 false)
ret void
}
define void @memset_p3_sz2048_align_4_varsetval(ptr addrspace(3) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p3_sz2048_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v1, v1, s0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-NEXT: v_mov_b32_e32 v1, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-NEXT: .LBB9_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-NEXT: ds_write2_b32 v0, v32, v31 offset0:30 offset1:31
-; GFX942-NEXT: ds_write2_b32 v0, v30, v29 offset0:28 offset1:29
-; GFX942-NEXT: ds_write2_b32 v0, v28, v27 offset0:26 offset1:27
-; GFX942-NEXT: ds_write2_b32 v0, v26, v25 offset0:24 offset1:25
-; GFX942-NEXT: ds_write2_b32 v0, v24, v23 offset0:22 offset1:23
-; GFX942-NEXT: ds_write2_b32 v0, v22, v21 offset0:20 offset1:21
-; GFX942-NEXT: ds_write2_b32 v0, v20, v19 offset0:18 offset1:19
-; GFX942-NEXT: ds_write2_b32 v0, v18, v17 offset0:16 offset1:17
-; GFX942-NEXT: ds_write2_b32 v0, v16, v15 offset0:14 offset1:15
-; GFX942-NEXT: ds_write2_b32 v0, v14, v13 offset0:12 offset1:13
-; GFX942-NEXT: ds_write2_b32 v0, v12, v11 offset0:10 offset1:11
-; GFX942-NEXT: ds_write2_b32 v0, v10, v9 offset0:8 offset1:9
-; GFX942-NEXT: ds_write2_b32 v0, v8, v7 offset0:6 offset1:7
-; GFX942-NEXT: ds_write2_b32 v0, v6, v5 offset0:4 offset1:5
-; GFX942-NEXT: ds_write2_b32 v0, v4, v3 offset0:2 offset1:3
-; GFX942-NEXT: ds_write2_b32 v0, v2, v1 offset1:1
-; GFX942-NEXT: ds_write2_b32 v0, v32, v31 offset0:62 offset1:63
-; GFX942-NEXT: ds_write2_b32 v0, v30, v29 offset0:60 offset1:61
-; GFX942-NEXT: ds_write2_b32 v0, v28, v27 offset0:58 offset1:59
-; GFX942-NEXT: ds_write2_b32 v0, v26, v25 offset0:56 offset1:57
-; GFX942-NEXT: ds_write2_b32 v0, v24, v23 offset0:54 offset1:55
-; GFX942-NEXT: ds_write2_b32 v0, v22, v21 offset0:52 offset1:53
-; GFX942-NEXT: ds_write2_b32 v0, v20, v19 offset0:50 offset1:51
-; GFX942-NEXT: ds_write2_b32 v0, v18, v17 offset0:48 offset1:49
-; GFX942-NEXT: ds_write2_b32 v0, v16, v15 offset0:46 offset1:47
-; GFX942-NEXT: ds_write2_b32 v0, v14, v13 offset0:44 offset1:45
-; GFX942-NEXT: ds_write2_b32 v0, v12, v11 offset0:42 offset1:43
-; GFX942-NEXT: ds_write2_b32 v0, v10, v9 offset0:40 offset1:41
-; GFX942-NEXT: ds_write2_b32 v0, v8, v7 offset0:38 offset1:39
-; GFX942-NEXT: ds_write2_b32 v0, v6, v5 offset0:36 offset1:37
-; GFX942-NEXT: ds_write2_b32 v0, v4, v3 offset0:34 offset1:35
-; GFX942-NEXT: ds_write2_b32 v0, v2, v1 offset0:32 offset1:33
-; GFX942-NEXT: v_add_u32_e32 v0, 0x100, v0
-; GFX942-NEXT: s_cbranch_vccnz .LBB9_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p3_sz2048_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: .LBB9_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:30 offset1:31
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:28 offset1:29
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:26 offset1:27
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:24 offset1:25
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:22 offset1:23
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:20 offset1:21
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:18 offset1:19
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:16 offset1:17
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:14 offset1:15
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:12 offset1:13
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:10 offset1:11
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:8 offset1:9
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:6 offset1:7
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:4 offset1:5
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:2 offset1:3
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset1:1
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v32, v31 offset0:62 offset1:63
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v30, v29 offset0:60 offset1:61
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v28, v27 offset0:58 offset1:59
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v26, v25 offset0:56 offset1:57
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v24, v23 offset0:54 offset1:55
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v22, v21 offset0:52 offset1:53
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v20, v19 offset0:50 offset1:51
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v18, v17 offset0:48 offset1:49
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v16, v15 offset0:46 offset1:47
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v14, v13 offset0:44 offset1:45
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v12, v11 offset0:42 offset1:43
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v10, v9 offset0:40 offset1:41
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v8, v7 offset0:38 offset1:39
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v6, v5 offset0:36 offset1:37
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v4, v3 offset0:34 offset1:35
+; GFX942-SDAG-NEXT: ds_write2_b32 v0, v2, v1 offset0:32 offset1:33
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB9_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p3_sz2048_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v2, v1, 8, v1
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v2, v3, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x800
+; GFX942-GISEL-NEXT: .LBB9_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:2 offset1:3
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:4 offset1:5
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:6 offset1:7
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:8 offset1:9
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:10 offset1:11
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:12 offset1:13
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:14 offset1:15
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:16 offset1:17
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:18 offset1:19
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:20 offset1:21
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:22 offset1:23
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:24 offset1:25
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:26 offset1:27
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:28 offset1:29
+; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:30 offset1:31
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 0x100, v0
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB9_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p3.i64(ptr addrspace(3) noundef nonnull align 4 %dst, i8 %setval, i64 2048, i1 false)
ret void
}
define void @memset_p5_sz1055_align_4_varsetval(ptr addrspace(5) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p5_sz1055_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v1, v1, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_mov_b64_e32 v[34:35], 0x400
-; GFX942-NEXT: v_mov_b32_e32 v36, v0
-; GFX942-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:112
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:96
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:80
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:64
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:48
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:32
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:16
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[2:5], off
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:240
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:224
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:208
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:192
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:176
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:160
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:144
-; GFX942-NEXT: scratch_store_dwordx4 v36, v[2:5], off offset:128
-; GFX942-NEXT: v_add_u32_e32 v36, 0x100, v36
-; GFX942-NEXT: s_cbranch_vccnz .LBB10_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v1, v1, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:1024
-; GFX942-NEXT: scratch_store_dwordx2 v0, v[2:3], off offset:1040
-; GFX942-NEXT: scratch_store_dword v0, v2, off offset:1048
-; GFX942-NEXT: v_lshlrev_b16_e32 v2, 8, v1
-; GFX942-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-NEXT: scratch_store_short v0, v2, off offset:1052
-; GFX942-NEXT: scratch_store_byte v0, v1, off offset:1054
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p5_sz1055_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x400
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v36, v0
+; GFX942-SDAG-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:112
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:96
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:80
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:64
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:48
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:32
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:16
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[2:5], off
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[30:33], off offset:240
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[26:29], off offset:224
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[22:25], off offset:208
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[18:21], off offset:192
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[14:17], off offset:176
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[10:13], off offset:160
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[6:9], off offset:144
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v36, v[2:5], off offset:128
+; GFX942-SDAG-NEXT: v_add_u32_e32 v36, 0x100, v36
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB10_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:1024
+; GFX942-SDAG-NEXT: scratch_store_dwordx2 v0, v[2:3], off offset:1040
+; GFX942-SDAG-NEXT: scratch_store_dword v0, v2, off offset:1048
+; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v2, 8, v1
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: scratch_store_short v0, v2, off offset:1052
+; GFX942-SDAG-NEXT: scratch_store_byte v0, v1, off offset:1054
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p5_sz1055_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v3, v2, 8, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v3, v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x400
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v0
+; GFX942-GISEL-NEXT: .LBB10_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:16
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:32
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:48
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:64
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:80
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:96
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:112
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:128
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:144
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:160
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:176
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:192
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:208
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:224
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v8, v[2:5], off offset:240
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v8, 0x100, v8
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB10_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v3, v2, 8, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v4, 16, v2
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v2, 24, v2
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v3, v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:1024
+; GFX942-GISEL-NEXT: s_nop 1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
+; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-GISEL-NEXT: scratch_store_dwordx2 v0, v[4:5], off offset:1040
+; GFX942-GISEL-NEXT: scratch_store_dword v0, v2, off offset:1048
+; GFX942-GISEL-NEXT: scratch_store_short v0, v3, off offset:1052
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1054
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p5.i64(ptr addrspace(5) noundef nonnull align 4 %dst, i8 %setval, i64 1055, i1 false)
ret void
}
define void @memset_p5_sz2048_align_4_varsetval(ptr addrspace(5) align 4 %dst, i8 %setval) #0 {
-; GFX942-LABEL: memset_p5_sz2048_align_4_varsetval:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, 0x4040404
-; GFX942-NEXT: v_perm_b32 v2, v1, v1, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, v2
-; GFX942-NEXT: v_mov_b32_e32 v4, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, v2
-; GFX942-NEXT: v_mov_b32_e32 v6, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v2
-; GFX942-NEXT: v_mov_b32_e32 v8, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, v2
-; GFX942-NEXT: v_mov_b32_e32 v10, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v2
-; GFX942-NEXT: v_mov_b32_e32 v12, v2
-; GFX942-NEXT: v_mov_b32_e32 v13, v2
-; GFX942-NEXT: v_mov_b32_e32 v14, v2
-; GFX942-NEXT: v_mov_b32_e32 v15, v2
-; GFX942-NEXT: v_mov_b32_e32 v16, v2
-; GFX942-NEXT: v_mov_b32_e32 v17, v2
-; GFX942-NEXT: v_mov_b32_e32 v18, v2
-; GFX942-NEXT: v_mov_b32_e32 v19, v2
-; GFX942-NEXT: v_mov_b32_e32 v20, v2
-; GFX942-NEXT: v_mov_b32_e32 v21, v2
-; GFX942-NEXT: v_mov_b32_e32 v22, v2
-; GFX942-NEXT: v_mov_b32_e32 v23, v2
-; GFX942-NEXT: v_mov_b32_e32 v24, v2
-; GFX942-NEXT: v_mov_b32_e32 v25, v2
-; GFX942-NEXT: v_mov_b32_e32 v26, v2
-; GFX942-NEXT: v_mov_b32_e32 v27, v2
-; GFX942-NEXT: v_mov_b32_e32 v28, v2
-; GFX942-NEXT: v_mov_b32_e32 v29, v2
-; GFX942-NEXT: v_mov_b32_e32 v30, v2
-; GFX942-NEXT: v_mov_b32_e32 v31, v2
-; GFX942-NEXT: v_mov_b32_e32 v32, v2
-; GFX942-NEXT: v_mov_b32_e32 v33, v2
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_mov_b64_e32 v[34:35], 0x800
-; GFX942-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_add_u32 s0, s0, 0x100
-; GFX942-NEXT: s_addc_u32 s1, s1, 0
-; GFX942-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:112
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:96
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:80
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:64
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[2:5], off
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:240
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:224
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:208
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:192
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:176
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:160
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:144
-; GFX942-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:128
-; GFX942-NEXT: v_add_u32_e32 v0, 0x100, v0
-; GFX942-NEXT: s_cbranch_vccnz .LBB11_1
-; GFX942-NEXT: ; %bb.2: ; %static-memset-post-expansion
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p5_sz2048_align_4_varsetval:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
+; GFX942-SDAG-NEXT: v_perm_b32 v2, v1, v1, s0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v12, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v13, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v14, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v17, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v24, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v25, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v26, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v27, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v28, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v29, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v30, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v31, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v32, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v33, v2
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[34:35], 0x800
+; GFX942-SDAG-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-SDAG-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[34:35]
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:112
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:96
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:80
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:64
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:48
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:32
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:16
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[30:33], off offset:240
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[26:29], off offset:224
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[22:25], off offset:208
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[18:21], off offset:192
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[14:17], off offset:176
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[10:13], off offset:160
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[6:9], off offset:144
+; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:128
+; GFX942-SDAG-NEXT: v_add_u32_e32 v0, 0x100, v0
+; GFX942-SDAG-NEXT: s_cbranch_vccnz .LBB11_1
+; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p5_sz2048_align_4_varsetval:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v2, v1, 8, v1
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v1
+; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX942-GISEL-NEXT: v_or3_b32 v2, v2, v3, v1
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], 0x800
+; GFX942-GISEL-NEXT: .LBB11_1: ; %static-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 0x100
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, s[0:1], v[6:7]
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:16
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:32
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:48
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:64
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:80
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:96
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:112
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:128
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:144
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:160
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:176
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:192
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:208
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:224
+; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:240
+; GFX942-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
+; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 0x100, v0
+; GFX942-GISEL-NEXT: s_cbranch_scc1 .LBB11_1
+; GFX942-GISEL-NEXT: ; %bb.2: ; %static-memset-post-expansion
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p5.i64(ptr addrspace(5) noundef nonnull align 4 %dst, i8 %setval, i64 2048, i1 false)
ret void
}
define void @memset_p1_varsz_align_4_set40(ptr addrspace(1) align 4 %dst, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsz_align_4_set40:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v10, -16, v2
-; GFX942-NEXT: v_mov_b32_e32 v11, v3
-; GFX942-NEXT: v_and_b32_e32 v8, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v9, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v4, 0x28282828
-; GFX942-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-NEXT: v_mov_b32_e32 v6, v4
-; GFX942-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_2
-; GFX942-NEXT: .LBB12_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB12_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 40
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[4:5], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB12_5
-; GFX942-NEXT: .LBB12_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set40:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v10, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, 0x28282828
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[10:11]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-SDAG-NEXT: .LBB12_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 40
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[4:5], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[8:9]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-SDAG-NEXT: .LBB12_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set40:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0x28282828
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB12_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_2
+; GFX942-GISEL-NEXT: .LBB12_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB12_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 4, v[2:3]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 40
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB12_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB12_5
+; GFX942-GISEL-NEXT: .LBB12_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 40, i64 %size, i1 false)
ret void
}
define void @memset_p1_varsz_align_4_set0(ptr addrspace(1) align 4 %dst, i64 %size) #0 {
-; GFX942-LABEL: memset_p1_varsz_align_4_set0:
-; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: v_and_b32_e32 v6, -16, v2
-; GFX942-NEXT: v_mov_b32_e32 v7, v3
-; GFX942-NEXT: v_and_b32_e32 v4, 15, v2
-; GFX942-NEXT: v_mov_b32_e32 v5, 0
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_3
-; GFX942-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
-; GFX942-NEXT: v_mov_b32_e32 v8, v5
-; GFX942-NEXT: v_mov_b32_e32 v9, v5
-; GFX942-NEXT: v_mov_b32_e32 v10, v5
-; GFX942-NEXT: v_mov_b32_e32 v11, v5
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 16
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_2
-; GFX942-NEXT: .LBB13_3: ; %Flow4
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_mov_b64 s[0:1], 0
-; GFX942-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX942-NEXT: s_cbranch_execz .LBB13_6
-; GFX942-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
-; GFX942-NEXT: v_and_b32_e32 v2, -16, v2
-; GFX942-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
-; GFX942-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-NEXT: s_mov_b64 s[4:5], 0
-; GFX942-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
-; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
-; GFX942-NEXT: s_add_u32 s4, s4, 1
-; GFX942-NEXT: s_addc_u32 s5, s5, 0
-; GFX942-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
-; GFX942-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX942-NEXT: global_store_byte v[6:7], v2, off
-; GFX942-NEXT: s_andn2_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_cbranch_execnz .LBB13_5
-; GFX942-NEXT: .LBB13_6: ; %Flow2
-; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_setpc_b64 s[30:31]
+; GFX942-SDAG-LABEL: memset_p1_varsz_align_4_set0:
+; GFX942-SDAG: ; %bb.0: ; %entry
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-SDAG-NEXT: v_and_b32_e32 v6, -16, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v3
+; GFX942-SDAG-NEXT: v_and_b32_e32 v4, 15, v2
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-SDAG-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v8, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v9, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, v5
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, v5
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[12:13], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 16
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[6:7]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[12:13], v[8:11], off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-SDAG-NEXT: .LBB13_3: ; %Flow4
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
+; GFX942-SDAG-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-SDAG-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-SDAG-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-SDAG-NEXT: v_and_b32_e32 v2, -16, v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 0, v[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-SDAG-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-SDAG-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[4:5]
+; GFX942-SDAG-NEXT: s_add_u32 s4, s4, 1
+; GFX942-SDAG-NEXT: s_addc_u32 s5, s5, 0
+; GFX942-SDAG-NEXT: v_cmp_ge_u64_e32 vcc, s[4:5], v[4:5]
+; GFX942-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX942-SDAG-NEXT: global_store_byte v[6:7], v2, off
+; GFX942-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX942-SDAG-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-SDAG-NEXT: .LBB13_6: ; %Flow2
+; GFX942-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
+; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX942-GISEL-LABEL: memset_p1_varsz_align_4_set0:
+; GFX942-GISEL: ; %bb.0: ; %entry
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], 15
+; GFX942-GISEL-NEXT: v_and_b32_e32 v8, 15, v2
+; GFX942-GISEL-NEXT: v_sub_co_u32_e32 v10, vcc, v2, v8
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_subbrev_co_u32_e32 v11, vcc, 0, v3, vcc
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_3
+; GFX942-GISEL-NEXT: ; %bb.1: ; %dynamic-memset-expansion-main-body.preheader
+; GFX942-GISEL-NEXT: s_mov_b32 s4, 0
+; GFX942-GISEL-NEXT: s_mov_b32 s5, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s6, s4
+; GFX942-GISEL-NEXT: s_mov_b32 s7, s4
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB13_2: ; %dynamic-memset-expansion-main-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[12:13], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 16
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v12, vcc, v0, v12
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v13, vcc, v1, v13, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[10:11]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[12:13], v[4:7], off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_2
+; GFX942-GISEL-NEXT: .LBB13_3: ; %Flow4
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_mov_b64 s[0:1], 0
+; GFX942-GISEL-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GFX942-GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX942-GISEL-NEXT: s_cbranch_execz .LBB13_6
+; GFX942-GISEL-NEXT: ; %bb.4: ; %dynamic-memset-expansion-residual-body.preheader
+; GFX942-GISEL-NEXT: v_lshrrev_b64 v[2:3], 4, v[2:3]
+; GFX942-GISEL-NEXT: v_lshl_add_u64 v[0:1], v[2:3], 4, v[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX942-GISEL-NEXT: s_mov_b64 s[4:5], 0
+; GFX942-GISEL-NEXT: .LBB13_5: ; %dynamic-memset-expansion-residual-body
+; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
+; GFX942-GISEL-NEXT: s_add_u32 s0, s0, 1
+; GFX942-GISEL-NEXT: v_add_co_u32_e32 v4, vcc, v0, v4
+; GFX942-GISEL-NEXT: s_addc_u32 s1, s1, 0
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v5, vcc, v1, v5, vcc
+; GFX942-GISEL-NEXT: v_cmp_ge_u64_e32 vcc, s[0:1], v[8:9]
+; GFX942-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX942-GISEL-NEXT: global_store_byte v[4:5], v2, off
+; GFX942-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; GFX942-GISEL-NEXT: s_cbranch_execnz .LBB13_5
+; GFX942-GISEL-NEXT: .LBB13_6: ; %Flow2
+; GFX942-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
+; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 4 %dst, i8 0, i64 %size, i1 false)
ret void
@@ -1105,5 +1894,4 @@ declare void @llvm.memset.p5.i64(ptr addrspace(5) noalias nocapture writeonly, i
attributes #0 = { nounwind }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX942-GISEL: {{.*}}
-; GFX942-SDAG: {{.*}}
+; GFX942: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/roundeven.ll b/llvm/test/CodeGen/AMDGPU/roundeven.ll
index bf40c1ef201f5..bd4879b46e28b 100644
--- a/llvm/test/CodeGen/AMDGPU/roundeven.ll
+++ b/llvm/test/CodeGen/AMDGPU/roundeven.ll
@@ -1,11 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=GFX6 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefix=GFX7 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefix=GFX8 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=GFX6 %s
+; RUN: llc -global-isel -mtriple=amdgpu7.01 < %s | FileCheck -check-prefix=GFX7 %s
+; RUN: llc -global-isel -mtriple=amdgpu8.03 < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgpu9.00 < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -global-isel -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=+real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-TRUE16 %s
-; RUN: llc -global-isel -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
+; RUN: llc -global-isel -mtriple=amdgpu11.00 -mattr=-real-true16 -amdgpu-enable-delay-alu=0 < %s | FileCheck -check-prefixes=GFX10PLUS,GFX11,GFX11-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefix=SDAG_GFX6 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu7.01 < %s | FileCheck -check-prefix=SDAG_GFX7 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03 < %s | FileCheck -check-prefix=SDAG_GFX8 %s
@@ -430,37 +430,43 @@ define <2 x half> @v_roundeven_v2f16(<2 x half> %x) {
; GFX6-LABEL: v_roundeven_v2f16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_rndne_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_rndne_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_roundeven_v2f16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: v_rndne_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_rndne_f32_e32 v0, v0
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX7-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_roundeven_v2f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rndne_f16_sdwa v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rndne_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_rndne_f16_e32 v1, v0
+; GFX8-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_roundeven_v2f16:
@@ -571,38 +577,45 @@ define <2 x half> @v_roundeven_v2f16_fneg(<2 x half> %x) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX6-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX6-NEXT: v_rndne_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_rndne_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v1, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_roundeven_v2f16_fneg:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
+; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
; GFX7-NEXT: v_rndne_f32_e32 v1, v1
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX7-NEXT: v_rndne_f32_e32 v0, v0
; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX7-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v0
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_roundeven_v2f16_fneg:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rndne_f16_sdwa v1, -v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rndne_f16_e64 v0, -v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-NEXT: v_rndne_f16_e32 v1, v0
+; GFX8-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_roundeven_v2f16_fneg:
@@ -719,58 +732,69 @@ define <4 x half> @v_roundeven_v4f16(<4 x half> %x) {
; GFX6-LABEL: v_roundeven_v4f16:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX6-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX6-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX6-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX6-NEXT: v_rndne_f32_e32 v0, v0
; GFX6-NEXT: v_rndne_f32_e32 v2, v2
-; GFX6-NEXT: v_rndne_f32_e32 v3, v3
+; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX6-NEXT: v_rndne_f32_e32 v3, v3
; GFX6-NEXT: v_rndne_f32_e32 v1, v1
-; GFX6-NEXT: v_rndne_f32_e32 v0, v0
-; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX6-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX6-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX6-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GFX6-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX6-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX6-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GFX6-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v3
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-LABEL: v_roundeven_v4f16:
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v0
-; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v2
+; GFX7-NEXT: v_cvt_f32_f16_e32 v2, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX7-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX7-NEXT: v_cvt_f32_f16_e32 v1, v1
-; GFX7-NEXT: v_cvt_f32_f16_e32 v0, v0
+; GFX7-NEXT: v_rndne_f32_e32 v0, v0
; GFX7-NEXT: v_rndne_f32_e32 v2, v2
-; GFX7-NEXT: v_rndne_f32_e32 v3, v3
+; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX7-NEXT: v_cvt_f16_f32_e32 v2, v2
-; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
+; GFX7-NEXT: v_rndne_f32_e32 v3, v3
; GFX7-NEXT: v_rndne_f32_e32 v1, v1
-; GFX7-NEXT: v_rndne_f32_e32 v0, v0
-; GFX7-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX7-NEXT: v_cvt_f16_f32_e32 v3, v3
; GFX7-NEXT: v_cvt_f16_f32_e32 v1, v1
+; GFX7-NEXT: v_bfe_u32 v0, v0, 0, 16
+; GFX7-NEXT: v_bfe_u32 v2, v2, 0, 16
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7-NEXT: v_bfe_u32 v2, v3, 0, 16
+; GFX7-NEXT: v_bfe_u32 v1, v1, 0, 16
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v3
; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: v_roundeven_v4f16:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_rndne_f16_sdwa v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rndne_f16_sdwa v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1
-; GFX8-NEXT: v_rndne_f16_e32 v1, v1
-; GFX8-NEXT: v_rndne_f16_e32 v0, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v3
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX8-NEXT: v_rndne_f16_e32 v2, v0
+; GFX8-NEXT: v_rndne_f16_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_rndne_f16_e32 v3, v1
+; GFX8-NEXT: v_rndne_f16_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; GFX8-NEXT: v_mov_b32_e32 v4, 16
+; GFX8-NEXT: v_lshlrev_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_lshlrev_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT: v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: v_roundeven_v4f16:
diff --git a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
index 7d5fca0333322..81a2db1fd52b7 100644
--- a/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/shrink-add-sub-constant.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -global-isel=0 -mtriple=amdgpu6.00 < %s | FileCheck --check-prefixes=SI,SI-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu6.00 < %s | FileCheck --check-prefixes=SI,SI-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck --check-prefixes=SI,SI-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.02 < %s | FileCheck --check-prefixes=VI,VI-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck --check-prefixes=VI,VI-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.02 < %s | FileCheck --check-prefixes=VI,VI-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck --check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck --check-prefixes=GFX10,GFX10-SDAG %s
@@ -1843,44 +1843,88 @@ define amdgpu_kernel void @v_test_i16_x_sub_64_multi_use(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_test_v2i16_x_sub_64_64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_64_64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v3, vcc, 64, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xffc00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_64_64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_subrev_i32_e32 v3, vcc, 64, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xffc00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_64_64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 64
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_sub_u16_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_subrev_u16_e32 v3, 64, v3
-; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_64_64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffffc0, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffffc0, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_64_64:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 64
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_sub_u16_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_subrev_u16_e32 v3, 64, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_64_64:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_not_b32_e32 v4, 63
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0xffc0, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_sub_64_64:
; GFX9: ; %bb.0:
@@ -1927,44 +1971,88 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_64(ptr addrspace(1) %out, ptr a
}
define amdgpu_kernel void @v_test_v2i16_x_sub_7_64(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_7_64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, -7, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xffc00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_7_64:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, -7, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xffc00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_7_64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 64
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_e32 v2, -7, v3
-; VI-NEXT: v_sub_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_7_64:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffffc0, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, -7, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_7_64:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 64
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_e32 v2, -7, v3
+; VI-SDAG-NEXT: v_sub_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_7_64:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_not_b32_e32 v4, 63
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, -7, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_sub_7_64:
; GFX9-SDAG: ; %bb.0:
@@ -2024,44 +2112,88 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_7_64(ptr addrspace(1) %out, ptr ad
}
define amdgpu_kernel void @v_test_v2i16_x_sub_64_123(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_64_123:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v3, vcc, 64, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xff850000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_64_123:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_subrev_i32_e32 v3, vcc, 64, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xff850000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_64_123:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0xffffff85
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_subrev_u16_e32 v3, 64, v3
-; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_64_123:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffff85, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffffc0, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_64_123:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0xffffff85
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_subrev_u16_e32 v3, 64, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_64_123:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0xffffff85
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0xffc0, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_sub_64_123:
; GFX9-SDAG: ; %bb.0:
@@ -2122,42 +2254,84 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_64_123(ptr addrspace(1) %out, ptr
; Can fold 0 and inline immediate in other half.
define amdgpu_kernel void @v_test_v2i16_x_sub_7_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_7_0:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, -7, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v3, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_7_0:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, -7, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v3, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_7_0:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; VI-NEXT: v_add_u16_e32 v3, -7, v3
-; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_7_0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, -7, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_7_0:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; VI-SDAG-NEXT: v_add_u16_e32 v3, -7, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_7_0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; VI-GISEL-NEXT: v_add_u16_e32 v3, -7, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_sub_7_0:
; GFX9: ; %bb.0:
@@ -2205,40 +2379,82 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_7_0(ptr addrspace(1) %out, ptr add
; Can fold 0 and inline immediate in other half.
define amdgpu_kernel void @v_test_v2i16_x_sub_0_16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_0_16:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_0_16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_0_16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_mov_b32_e32 v2, -16
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_0_16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, -16, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_0_16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, -16
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_0_16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, -16
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_sub_0_16:
; GFX9: ; %bb.0:
@@ -2285,40 +2501,82 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_16(ptr addrspace(1) %out, ptr ad
}
define amdgpu_kernel void @v_test_v2i16_x_sub_0_1_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_0_1_0:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0x3c000000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_0_1_0:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0x3c000000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_0_1_0:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_mov_b32_e32 v2, 0x3c00
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_0_1_0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0x3c00, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_0_1_0:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x3c00
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_0_1_0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0x3c00
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_sub_0_1_0:
; GFX9-SDAG: ; %bb.0:
@@ -2378,40 +2636,82 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_1_0(ptr addrspace(1) %out, ptr a
}
define amdgpu_kernel void @v_test_v2i16_x_sub_0_neg1_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_sub_0_neg1_0:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xbc000000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_sub_0_neg1_0:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xbc000000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_sub_0_neg1_0:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_mov_b32_e32 v2, 0xffffbc00
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_sub_0_neg1_0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffbc00, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_sub_0_neg1_0:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0xffffbc00
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_sub_0_neg1_0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, 0xffffbc00
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_sub_0_neg1_0:
; GFX9-SDAG: ; %bb.0:
@@ -2472,44 +2772,88 @@ define amdgpu_kernel void @v_test_v2i16_x_sub_0_neg1_0(ptr addrspace(1) %out, pt
; -32 isn't an inline immediate, but 32 is
define amdgpu_kernel void @v_test_v2i16_x_add_neg32_neg32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg32_neg32:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v3, vcc, 32, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xffe00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg32_neg32:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_subrev_i32_e32 v3, vcc, 32, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xffe00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg32_neg32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 32
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_sub_u16_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_subrev_u16_e32 v3, 32, v3
-; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg32_neg32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffffe0, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffffe0, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg32_neg32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 32
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_sub_u16_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_subrev_u16_e32 v3, 32, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg32_neg32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_not_b32_e32 v4, 31
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0xffe0, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_neg32_neg32:
; GFX9: ; %bb.0:
@@ -2556,40 +2900,82 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_neg32(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_test_v2i16_x_add_0_neg32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_0_neg32:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xffe00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_0_neg32:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xffe00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_0_neg32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_mov_b32_e32 v2, 32
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_sub_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_0_neg32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffffe0, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_0_neg32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 32
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_sub_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_0_neg32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_not_b32_e32 v2, 31
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_0_neg32:
; GFX9: ; %bb.0:
@@ -2636,42 +3022,84 @@ define amdgpu_kernel void @v_test_v2i16_x_add_0_neg32(ptr addrspace(1) %out, ptr
}
define amdgpu_kernel void @v_test_v2i16_x_add_neg32_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg32_0:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_subrev_i32_e32 v3, vcc, 32, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v3, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg32_0:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_subrev_i32_e32 v3, vcc, 32, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v3, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg32_0:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; VI-NEXT: v_subrev_u16_e32 v3, 32, v3
-; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg32_0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffffe0, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg32_0:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; VI-SDAG-NEXT: v_subrev_u16_e32 v3, 32, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg32_0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; VI-GISEL-NEXT: v_add_u16_e32 v3, 0xffe0, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_neg32_0:
; GFX9: ; %bb.0:
@@ -2719,44 +3147,88 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_0(ptr addrspace(1) %out, ptr
; 16 and -16 are both inline immediates
define amdgpu_kernel void @v_test_v2i16_x_add_neg16_neg16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg16_neg16:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, -16, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg16_neg16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, -16, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg16_neg16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, -16, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, -16, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg16_neg16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, -16
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_e32 v2, -16, v3
+; VI-SDAG-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg16_neg16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, -16
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_e32 v2, -16, v3
-; VI-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg16_neg16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, -16
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, -16, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_neg16_neg16:
; GFX9: ; %bb.0:
@@ -2803,40 +3275,82 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg16_neg16(ptr addrspace(1) %out,
}
define amdgpu_kernel void @v_test_v2i16_x_add_0_neg16(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_0_neg16:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_0_neg16:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_0_neg16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_mov_b32_e32 v2, -16
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_0_neg16:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, -16, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_0_neg16:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, -16
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_0_neg16:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v2, -16
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_0_neg16:
; GFX9: ; %bb.0:
@@ -2883,42 +3397,84 @@ define amdgpu_kernel void @v_test_v2i16_x_add_0_neg16(ptr addrspace(1) %out, ptr
}
define amdgpu_kernel void @v_test_v2i16_x_add_neg16_0(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg16_0:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, -16, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v3, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg16_0:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, -16, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v3, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg16_0:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
-; VI-NEXT: v_add_u16_e32 v3, -16, v3
-; VI-NEXT: v_or_b32_e32 v2, v3, v2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg16_0:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, -16, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg16_0:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xffff0000, v3
+; VI-SDAG-NEXT: v_add_u16_e32 v3, -16, v3
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg16_0:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v3
+; VI-GISEL-NEXT: v_add_u16_e32 v3, -16, v3
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v3, v2
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_neg16_0:
; GFX9: ; %bb.0:
@@ -2965,44 +3521,88 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg16_0(ptr addrspace(1) %out, ptr
}
define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg_fpone:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0xffffc400, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xc4000000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, 0xffffc400, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xc4000000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg_fpone:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0xffffc400
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_e32 v2, 0xc400, v3
-; VI-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffc400, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffc400, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0xffffc400
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_e32 v2, 0xc400, v3
+; VI-SDAG-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg_fpone:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0xffffc400
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0xc400, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_add_neg_fpone:
; GFX9-SDAG: ; %bb.0:
@@ -3086,44 +3686,88 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fpone(ptr addrspace(1) %out, p
}
define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg_negfpone:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x4400, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0x44000000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, 0x4400, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0x44000000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
+;
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0x4400, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x4400, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x4400
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_e32 v2, 0x4400, v3
+; VI-SDAG-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg_negfpone:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0x4400
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_e32 v2, 0x4400, v3
-; VI-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg_negfpone:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x4400
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0x4400, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_add_neg_negfpone:
; GFX9-SDAG: ; %bb.0:
@@ -3207,44 +3851,88 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfpone(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_test_v2i16_x_add_neg_fptwo(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg_fptwo:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0x4000, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, 2.0, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg_fptwo:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, 0x4000, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 2.0, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg_fptwo:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0x4000
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_e32 v2, 0x4000, v3
-; VI-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg_fptwo:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0x4000, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0x4000, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg_fptwo:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x4000
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_e32 v2, 0x4000, v3
+; VI-SDAG-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg_fptwo:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0x4000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0x4000, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_neg_fptwo:
; GFX9: ; %bb.0:
@@ -3291,44 +3979,88 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_fptwo(ptr addrspace(1) %out, p
}
define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfptwo(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_neg_negfptwo:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s4, 0xffff0000
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_add_i32_e32 v3, vcc, 0xffffc000, v2
-; SI-NEXT: v_bfi_b32 v2, s4, v2, v3
-; SI-NEXT: v_add_i32_e32 v2, vcc, -2.0, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_neg_negfptwo:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b32 s4, 0xffff0000
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_add_i32_e32 v3, vcc, 0xffffc000, v2
+; SI-SDAG-NEXT: v_bfi_b32 v2, s4, v2, v3
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, -2.0, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg_negfptwo:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: v_mov_b32_e32 v4, 0xffffc000
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_add_u16_e32 v2, 0xc000, v3
-; VI-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v2, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_neg_negfptwo:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v3, vcc, 0xffffc000, v3
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffc000, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; SI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg_negfptwo:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0xffffc000
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_add_u16_e32 v2, 0xc000, v3
+; VI-SDAG-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg_negfptwo:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v4, 0xffffc000
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0xc000, v3
+; VI-GISEL-NEXT: v_add_u16_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, v2, v3
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_neg_negfptwo:
; GFX9: ; %bb.0:
@@ -3375,40 +4107,81 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg_negfptwo(ptr addrspace(1) %out
}
define amdgpu_kernel void @v_test_v2i16_x_add_undef_neg32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; SI-LABEL: v_test_v2i16_x_add_undef_neg32:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b64 s[2:3], s[6:7]
-; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; SI-NEXT: v_add_i32_e32 v2, vcc, 0xffe00000, v2
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_endpgm
+; SI-SDAG-LABEL: v_test_v2i16_x_add_undef_neg32:
+; SI-SDAG: ; %bb.0:
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, 0
+; SI-SDAG-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-SDAG-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-SDAG-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; SI-SDAG-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
+; SI-SDAG-NEXT: v_add_i32_e32 v2, vcc, 0xffe00000, v2
+; SI-SDAG-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-SDAG-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_undef_neg32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_mov_b32_e32 v2, 32
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_sub_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; SI-GISEL-LABEL: v_test_v2i16_x_add_undef_neg32:
+; SI-GISEL: ; %bb.0:
+; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
+; SI-GISEL-NEXT: v_mov_b32_e32 v1, 0
+; SI-GISEL-NEXT: s_mov_b32 s6, 0
+; SI-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-GISEL-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-GISEL-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-GISEL-NEXT: s_mov_b64 s[2:3], s[6:7]
+; SI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; SI-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: v_add_i32_e32 v2, vcc, 0xffffffe0, v2
+; SI-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; SI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_test_v2i16_x_add_undef_neg32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 32
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_sub_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_undef_neg32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_not_b32_e32 v2, 31
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: s_and_b32 s0, 0xffff, s0
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; VI-GISEL-NEXT: v_or_b32_e32 v2, s0, v2
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-LABEL: v_test_v2i16_x_add_undef_neg32:
; GFX9: ; %bb.0:
@@ -3516,22 +4289,44 @@ define amdgpu_kernel void @v_test_v2i16_x_add_neg32_undef(ptr addrspace(1) %out,
; SI-GISEL-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: v_test_v2i16_x_add_neg32_undef:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
-; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_subrev_u16_e32 v2, 32, v3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: v_test_v2i16_x_add_neg32_undef:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s2, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: flat_load_dword v3, v[0:1]
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_add_u32_e32 v0, vcc, s0, v2
+; VI-SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDAG-NEXT: s_waitcnt vmcnt(0)
+; VI-SDAG-NEXT: v_subrev_u16_e32 v2, 32, v3
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: v_test_v2i16_x_add_neg32_undef:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: v_lshlrev_b32_e32 v2, 2, v0
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: flat_load_dword v3, v[0:1]
+; VI-GISEL-NEXT: s_and_b32 s2, 0xffff, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: v_add_u32_e32 v0, vcc, v0, v2
+; VI-GISEL-NEXT: s_lshl_b32 s0, s2, 16
+; VI-GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-GISEL-NEXT: s_waitcnt vmcnt(0)
+; VI-GISEL-NEXT: v_add_u16_e32 v2, 0xffe0, v3
+; VI-GISEL-NEXT: v_or_b32_e32 v2, s0, v2
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: v_test_v2i16_x_add_neg32_undef:
; GFX9-SDAG: ; %bb.0:
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
index ba91e9230d81f..00c5429269df8 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fadd.f16.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX10,GFX10-GISEL %s
@@ -121,13 +121,23 @@ define <2 x half> @v_constained_fadd_v2f16_fpexcept_strict(<2 x half> %x, <2 x h
; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fadd_v2f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fadd_v2f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fadd_v2f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_constained_fadd_v2f16_fpexcept_strict:
; GFX10: ; %bb.0:
@@ -161,13 +171,23 @@ define <2 x half> @v_constained_fadd_v2f16_fpexcept_ignore(<2 x half> %x, <2 x h
; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fadd_v2f16_fpexcept_ignore:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fadd_v2f16_fpexcept_ignore:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fadd_v2f16_fpexcept_ignore:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_constained_fadd_v2f16_fpexcept_ignore:
; GFX10: ; %bb.0:
@@ -201,13 +221,23 @@ define <2 x half> @v_constained_fadd_v2f16_fpexcept_maytrap(<2 x half> %x, <2 x
; GFX9-NEXT: v_pk_add_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fadd_v2f16_fpexcept_maytrap:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fadd_v2f16_fpexcept_maytrap:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fadd_v2f16_fpexcept_maytrap:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: v_constained_fadd_v2f16_fpexcept_maytrap:
; GFX10: ; %bb.0:
@@ -249,14 +279,26 @@ define <3 x half> @v_constained_fadd_v3f16_fpexcept_strict(<3 x half> %x, <3 x h
; GFX9-GISEL-NEXT: v_pk_add_f16 v1, v1, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fadd_v3f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_add_f16_e32 v1, v1, v3
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fadd_v3f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_add_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fadd_v3f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_add_f16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fadd_v3f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -339,16 +381,30 @@ define <4 x half> @v_constained_fadd_v4f16_fpexcept_strict(<4 x half> %x, <4 x h
; GFX9-GISEL-NEXT: v_pk_add_f16 v1, v1, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fadd_v4f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_add_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_add_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_add_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fadd_v4f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_add_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_add_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fadd_v4f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_add_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fadd_v4f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -481,17 +537,34 @@ define amdgpu_ps <2 x half> @s_constained_fadd_v2f16_fpexcept_strict(<2 x half>
; GFX9-NEXT: v_pk_add_f16 v0, s2, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX8-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: s_lshr_b32 s1, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_add_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_add_f16_e32 v1, s2, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT: v_add_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-SDAG-NEXT: v_add_f16_e32 v1, s2, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-GISEL-NEXT: s_lshr_b32 s1, s3, 16
+; GFX8-GISEL-NEXT: v_add_f16_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-GISEL-NEXT: v_add_f16_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s1, s2, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-GISEL-NEXT: s_or_b32 s0, s1, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-LABEL: s_constained_fadd_v2f16_fpexcept_strict:
; GFX10: ; %bb.0:
@@ -529,6 +602,3 @@ declare <4 x half> @llvm.experimental.constrained.fadd.v4f16(<4 x half>, <4 x ha
attributes #0 = { strictfp }
attributes #1 = { inaccessiblememonly nounwind willreturn }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
index cc8abe6535b05..9eefa8501ac8c 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fmul.f16.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10PLUS,GFX10,GFX10-GISEL %s
@@ -205,13 +205,23 @@ define <2 x half> @v_constained_fmul_v2f16_fpexcept_strict(<2 x half> %x, <2 x h
; GFX9-NEXT: v_pk_mul_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fmul_v2f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fmul_v2f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fmul_v2f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_constained_fmul_v2f16_fpexcept_strict:
; GFX10PLUS: ; %bb.0:
@@ -239,13 +249,23 @@ define <2 x half> @v_constained_fmul_v2f16_fpexcept_ignore(<2 x half> %x, <2 x h
; GFX9-NEXT: v_pk_mul_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fmul_v2f16_fpexcept_ignore:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fmul_v2f16_fpexcept_ignore:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fmul_v2f16_fpexcept_ignore:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_constained_fmul_v2f16_fpexcept_ignore:
; GFX10PLUS: ; %bb.0:
@@ -273,13 +293,23 @@ define <2 x half> @v_constained_fmul_v2f16_fpexcept_maytrap(<2 x half> %x, <2 x
; GFX9-NEXT: v_pk_mul_f16 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fmul_v2f16_fpexcept_maytrap:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fmul_v2f16_fpexcept_maytrap:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fmul_v2f16_fpexcept_maytrap:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10PLUS-LABEL: v_constained_fmul_v2f16_fpexcept_maytrap:
; GFX10PLUS: ; %bb.0:
@@ -315,14 +345,26 @@ define <3 x half> @v_constained_fmul_v3f16_fpexcept_strict(<3 x half> %x, <3 x h
; GFX9-GISEL-NEXT: v_pk_mul_f16 v1, v1, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fmul_v3f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fmul_v3f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fmul_v3f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fmul_v3f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -411,16 +453,30 @@ define <4 x half> @v_constained_fmul_v4f16_fpexcept_strict(<4 x half> %x, <4 x h
; GFX9-GISEL-NEXT: v_pk_mul_f16 v1, v1, v3
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fmul_v4f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_mul_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_mul_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_mul_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fmul_v4f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fmul_v4f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v2, v1, v3
+; GFX8-GISEL-NEXT: v_mul_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fmul_v4f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -561,17 +617,34 @@ define amdgpu_ps <2 x half> @s_constained_fmul_v2f16_fpexcept_strict(<2 x half>
; GFX9-NEXT: v_pk_mul_f16 v0, s2, v0
; GFX9-NEXT: ; return to shader part epilog
;
-; GFX8-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: s_lshr_b32 s1, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mul_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mul_f16_e32 v1, s2, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT: v_mul_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-SDAG-NEXT: v_mul_f16_e32 v1, s2, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-GISEL-NEXT: s_lshr_b32 s1, s3, 16
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX8-GISEL-NEXT: v_mul_f16_e32 v0, s0, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s1, s2, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s0, s0, 16
+; GFX8-GISEL-NEXT: s_or_b32 s0, s1, s0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10PLUS-LABEL: s_constained_fmul_v2f16_fpexcept_strict:
; GFX10PLUS: ; %bb.0:
@@ -607,5 +680,4 @@ attributes #1 = { inaccessiblememonly nounwind willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11: {{.*}}
; GFX11-SDAG: {{.*}}
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
+; GFX8: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
index 5b61d49c7db5b..c8ff9b3a3d614 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_fsub.f16.ll
@@ -3,7 +3,7 @@
; RUN: llc -global-isel=1 -mtriple=amdgpu9.00-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.03-mesa-mesa3d < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu10.10-mesa-mesa3d < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
@@ -184,13 +184,24 @@ define <2 x half> @v_constained_fsub_v2f16_fpexcept_strict(<2 x half> %x, <2 x h
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -283,13 +294,24 @@ define <2 x half> @v_constained_fsub_v2f16_fpexcept_ignore(<2 x half> %x, <2 x h
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_ignore:
; GFX10-SDAG: ; %bb.0:
@@ -382,13 +404,24 @@ define <2 x half> @v_constained_fsub_v2f16_fpexcept_maytrap(<2 x half> %x, <2 x
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, v1 neg_lo:[0,1] neg_hi:[0,1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_f16_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v2, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v1, 0x80008000, v1
+; GFX8-GISEL-NEXT: v_add_f16_e32 v2, v0, v1
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fsub_v2f16_fpexcept_maytrap:
; GFX10-SDAG: ; %bb.0:
@@ -484,14 +517,27 @@ define <3 x half> @v_constained_fsub_v3f16_fpexcept_strict(<3 x half> %x, <3 x h
; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v4
-; GFX8-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v4, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-GISEL-NEXT: v_add_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_sub_f16_e32 v1, v1, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fsub_v3f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -614,16 +660,32 @@ define <4 x half> @v_constained_fsub_v4f16_fpexcept_strict(<4 x half> %x, <4 x h
; GFX9-GISEL-NEXT: v_pk_add_f16 v1, v1, v3 neg_lo:[0,1] neg_hi:[0,1]
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_sub_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
-; GFX8-NEXT: v_sub_f16_e32 v1, v1, v3
-; GFX8-NEXT: v_sub_f16_e32 v0, v0, v2
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v1, v4
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v4, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v5, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v1, v1, v3
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v2, 0x80008000, v2
+; GFX8-GISEL-NEXT: v_add_f16_e32 v4, v0, v2
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v3, 0x80008000, v3
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_add_f16_e32 v4, v1, v3
+; GFX8-GISEL-NEXT: v_add_f16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: v_constained_fsub_v4f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -791,17 +853,37 @@ define amdgpu_ps <2 x half> @s_constained_fsub_v2f16_fpexcept_strict(<2 x half>
; GFX9-GISEL-NEXT: v_pk_add_f16 v0, v0, s3 neg_lo:[0,1] neg_hi:[0,1]
; GFX9-GISEL-NEXT: ; return to shader part epilog
;
-; GFX8-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_lshr_b32 s0, s3, 16
-; GFX8-NEXT: s_lshr_b32 s1, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_sub_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_sub_f16_e32 v1, s2, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: ; return to shader part epilog
+; GFX8-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s3, 16
+; GFX8-SDAG-NEXT: s_lshr_b32 s1, s2, 16
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-SDAG-NEXT: v_sub_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-SDAG-NEXT: v_sub_f16_e32 v1, s2, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: ; return to shader part epilog
+;
+; GFX8-GISEL-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, 0x80008000, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: s_lshr_b32 s3, s0, 16
+; GFX8-GISEL-NEXT: v_add_f16_e32 v0, s2, v0
+; GFX8-GISEL-NEXT: s_lshr_b32 s1, s2, 16
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-GISEL-NEXT: v_add_f16_e32 v0, s1, v0
+; GFX8-GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX8-GISEL-NEXT: s_bfe_u32 s1, s1, 0x100000
+; GFX8-GISEL-NEXT: s_bfe_u32 s0, s0, 0x100000
+; GFX8-GISEL-NEXT: s_lshl_b32 s1, s1, 16
+; GFX8-GISEL-NEXT: s_or_b32 s0, s0, s1
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-GISEL-NEXT: ; return to shader part epilog
;
; GFX10-SDAG-LABEL: s_constained_fsub_v2f16_fpexcept_strict:
; GFX10-SDAG: ; %bb.0:
@@ -891,6 +973,5 @@ attributes #0 = { strictfp }
attributes #1 = { inaccessiblememonly nounwind willreturn }
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11: {{.*}}
-; GFX8-GISEL: {{.*}}
-; GFX8-SDAG: {{.*}}
+; GFX8: {{.*}}
; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll b/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
index 32f8ee68b3761..102aa44922e0b 100644
--- a/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/strict_ldexp.f16.ll
@@ -7,10 +7,10 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu6.00 < %s | FileCheck -check-prefixes=GCN,GFX6,GFX6-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.02 < %s | FileCheck -check-prefixes=GCN,GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.00 < %s | FileCheck -check-prefixes=GCN,GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GCN,GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; define half @test_ldexp_f16_i16(ptr addrspace(1) %out, half %a, i16 %b) #0 {
; %result = call half @llvm.experimental.constrained.ldexp.f16.i16(half %a, i16 %b, metadata !"round.dynamic", metadata !"fpexcept.strict")
@@ -99,17 +99,17 @@ define half @test_ldexp_f16_i32(ptr addrspace(1) %out, half %a, i32 %b) #0 {
; }
define <2 x half> @test_ldexp_v2f16_v2i32(ptr addrspace(1) %out, <2 x half> %a, <2 x i32> %b) #0 {
-; GFX8-LABEL: test_ldexp_v2f16_v2i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX8-NEXT: v_med3_i32 v1, v3, s4, v0
-; GFX8-NEXT: v_med3_i32 v0, v4, s4, v0
-; GFX8-NEXT: v_ldexp_f16_e32 v1, v2, v1
-; GFX8-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_ldexp_v2f16_v2i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v1, v3, s4, v0
+; GFX8-SDAG-NEXT: v_med3_i32 v0, v4, s4, v0
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v1, v2, v1
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v2f16_v2i32:
; GFX9-SDAG: ; %bb.0:
@@ -150,6 +150,20 @@ define <2 x half> @test_ldexp_v2f16_v2i32(ptr addrspace(1) %out, <2 x half> %a,
; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX8-GISEL-LABEL: test_ldexp_v2f16_v2i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v3, v3, v0, v1
+; GFX8-GISEL-NEXT: v_med3_i32 v0, v4, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 16
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v3, v2, v3
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v2f16_v2i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -194,19 +208,19 @@ define <2 x half> @test_ldexp_v2f16_v2i32(ptr addrspace(1) %out, <2 x half> %a,
}
define <3 x half> @test_ldexp_v3f16_v3i32(ptr addrspace(1) %out, <3 x half> %a, <3 x i32> %b) #0 {
-; GFX8-LABEL: test_ldexp_v3f16_v3i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v1, 0x7fff
-; GFX8-NEXT: v_med3_i32 v0, v4, s4, v1
-; GFX8-NEXT: v_med3_i32 v4, v5, s4, v1
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX8-NEXT: v_ldexp_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_med3_i32 v1, v6, s4, v1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v2
-; GFX8-NEXT: v_ldexp_f16_e32 v1, v3, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_ldexp_v3f16_v3i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v0, v4, s4, v1
+; GFX8-SDAG-NEXT: v_med3_i32 v4, v5, s4, v1
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v2, v2, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_med3_i32 v1, v6, s4, v1
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v2
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v1, v3, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v3f16_v3i32:
; GFX9-SDAG: ; %bb.0:
@@ -254,6 +268,23 @@ define <3 x half> @test_ldexp_v3f16_v3i32(ptr addrspace(1) %out, <3 x half> %a,
; GFX11-SDAG-FAKE16-NEXT: v_ldexp_f16_e32 v1, v3, v2
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX8-GISEL-LABEL: test_ldexp_v3f16_v3i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v4, v4, v0, v1
+; GFX8-GISEL-NEXT: v_med3_i32 v5, v5, v0, v1
+; GFX8-GISEL-NEXT: v_med3_i32 v0, v6, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v4, v2, v4
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v1, v3, v0
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_bfe_u32 v1, v1, 0, 16
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v3f16_v3i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -308,22 +339,22 @@ define <3 x half> @test_ldexp_v3f16_v3i32(ptr addrspace(1) %out, <3 x half> %a,
}
define <4 x half> @test_ldexp_v4f16_v4i32(ptr addrspace(1) %out, <4 x half> %a, <4 x i32> %b) #0 {
-; GFX8-LABEL: test_ldexp_v4f16_v4i32:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: s_movk_i32 s4, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x7fff
-; GFX8-NEXT: v_med3_i32 v1, v7, s4, v0
-; GFX8-NEXT: v_med3_i32 v6, v6, s4, v0
-; GFX8-NEXT: v_med3_i32 v5, v5, s4, v0
-; GFX8-NEXT: v_med3_i32 v0, v4, s4, v0
-; GFX8-NEXT: v_ldexp_f16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_ldexp_f16_e32 v3, v3, v6
-; GFX8-NEXT: v_ldexp_f16_sdwa v5, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT: v_ldexp_f16_e32 v0, v2, v0
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v5
-; GFX8-NEXT: v_or_b32_e32 v1, v3, v1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX8-SDAG-LABEL: test_ldexp_v4f16_v4i32:
+; GFX8-SDAG: ; %bb.0:
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: s_movk_i32 s4, 0x8000
+; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, 0x7fff
+; GFX8-SDAG-NEXT: v_med3_i32 v1, v7, s4, v0
+; GFX8-SDAG-NEXT: v_med3_i32 v6, v6, s4, v0
+; GFX8-SDAG-NEXT: v_med3_i32 v5, v5, s4, v0
+; GFX8-SDAG-NEXT: v_med3_i32 v0, v4, s4, v0
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v3, v3, v6
+; GFX8-SDAG-NEXT: v_ldexp_f16_sdwa v5, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-SDAG-NEXT: v_ldexp_f16_e32 v0, v2, v0
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX8-SDAG-NEXT: v_or_b32_e32 v1, v3, v1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_ldexp_v4f16_v4i32:
; GFX9-SDAG: ; %bb.0:
@@ -381,6 +412,26 @@ define <4 x half> @test_ldexp_v4f16_v4i32(ptr addrspace(1) %out, <4 x half> %a,
; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v1, v1, v3, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
+; GFX8-GISEL-LABEL: test_ldexp_v4f16_v4i32:
+; GFX8-GISEL: ; %bb.0:
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff8000
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, 0x7fff
+; GFX8-GISEL-NEXT: v_med3_i32 v4, v4, v0, v1
+; GFX8-GISEL-NEXT: v_med3_i32 v5, v5, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v4, v2, v4
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_med3_i32 v5, v6, v0, v1
+; GFX8-GISEL-NEXT: v_med3_i32 v0, v7, v0, v1
+; GFX8-GISEL-NEXT: v_ldexp_f16_e32 v5, v3, v5
+; GFX8-GISEL-NEXT: v_ldexp_f16_sdwa v1, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, 16
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v0, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_lshlrev_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v5, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
; GFX9-GISEL-LABEL: test_ldexp_v4f16_v4i32:
; GFX9-GISEL: ; %bb.0:
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
@@ -525,4 +576,5 @@ attributes #1 = { nocallback nofree nosync nounwind willreturn memory(inaccessib
; GFX11: {{.*}}
; GFX11-GISEL: {{.*}}
; GFX11-SDAG: {{.*}}
+; GFX8: {{.*}}
; GFX9: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
index 6a798a02ee217..dc9c7f3429464 100644
--- a/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_sat_pk_u8_i16.ll
@@ -6,12 +6,12 @@
; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=SDAG-GFX12,SDAG-GFX12-TRUE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=SDAG-GFX12,SDAG-GFX12-FAKE16 %s
-; RUN: llc -mtriple=amdgpu8.03 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-VI %s
-; RUN: llc -mtriple=amdgpu9.00 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-GFX9 %s
+; RUN: llc -mtriple=amdgpu8.03 -global-isel < %s | FileCheck -check-prefixes=GISEL-VI %s
+; RUN: llc -mtriple=amdgpu9.00 -global-isel < %s | FileCheck -check-prefixes=GISEL-GFX9 %s
; RUN: llc -mtriple=amdgpu11.01 -mattr=+real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-TRUE16 %s
-; RUN: llc -mtriple=amdgpu11.01 -mattr=-real-true16 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
+; RUN: llc -mtriple=amdgpu11.01 -mattr=-real-true16 -global-isel < %s | FileCheck -check-prefixes=GFX11,GISEL-GFX11,GISEL-GFX11-FAKE16 %s
; RUN: llc -mtriple=amdgpu12.00 -mattr=+real-true16 -global-isel < %s | FileCheck -check-prefixes=GISEL-GFX12,GISEL-GFX12-TRUE16 %s
-; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 -global-isel -global-isel-abort=2 < %s | FileCheck -check-prefixes=GISEL-GFX12,GISEL-GFX12-FAKE16 %s
+; RUN: llc -mtriple=amdgpu12.00 -mattr=-real-true16 -global-isel < %s | FileCheck -check-prefixes=GISEL-GFX12,GISEL-GFX12-FAKE16 %s
; <GFX9 has no V_SAT_PK, GFX9+ has V_SAT_PK, GFX11 has V_SAT_PK with t16
@@ -87,10 +87,10 @@ define <2 x i16> @basic_smax_smin(i16 %src0, i16 %src1) {
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
-; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v1
; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-VI-NEXT: v_min_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v1
; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
+; GISEL-VI-NEXT: v_min_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -235,14 +235,20 @@ define amdgpu_kernel void @basic_smax_smin_sgpr(ptr addrspace(1) %out, i32 inreg
; GISEL-VI-LABEL: basic_smax_smin_sgpr:
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GISEL-VI-NEXT: v_mov_b32_e32 v0, 0xff
; GISEL-VI-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-VI-NEXT: s_sext_i32_i16 s3, s3
; GISEL-VI-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-VI-NEXT: s_max_i32 s3, s3, 0
+; GISEL-VI-NEXT: s_max_i32 s2, s2, 0
; GISEL-VI-NEXT: s_sext_i32_i16 s3, s3
-; GISEL-VI-NEXT: v_med3_i32 v1, s2, 0, v0
-; GISEL-VI-NEXT: v_med3_i32 v0, s3, 0, v0
-; GISEL-VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-VI-NEXT: v_or_b32_e32 v2, v1, v0
+; GISEL-VI-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-VI-NEXT: s_min_i32 s3, s3, 0xff
+; GISEL-VI-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-VI-NEXT: s_and_b32 s3, 0xffff, s3
+; GISEL-VI-NEXT: s_and_b32 s2, 0xffff, s2
+; GISEL-VI-NEXT: s_lshl_b32 s3, s3, 16
+; GISEL-VI-NEXT: s_or_b32 s2, s2, s3
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, s2
; GISEL-VI-NEXT: v_mov_b32_e32 v0, s0
; GISEL-VI-NEXT: v_mov_b32_e32 v1, s1
; GISEL-VI-NEXT: flat_store_dword v[0:1], v2
@@ -384,8 +390,8 @@ define <2 x i16> @basic_smin_smax(i16 %src0, i16 %src1) {
; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
; GISEL-VI-NEXT: v_min_i16_e32 v1, 0xff, v1
; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0
-; GISEL-VI-NEXT: v_max_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
+; GISEL-VI-NEXT: v_max_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -515,11 +521,11 @@ define <2 x i16> @basic_smin_smax_combined(i16 %src0, i16 %src1) {
; GISEL-VI-LABEL: basic_smin_smax_combined:
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0xff
; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v1
-; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-VI-NEXT: v_min_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
+; GISEL-VI-NEXT: v_min_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
@@ -628,13 +634,13 @@ define <2 x i16> @vec_smax_smin(<2 x i16> %src) {
; GISEL-VI-LABEL: vec_smax_smin:
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-VI-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-VI-NEXT: v_max_i16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v0
+; GISEL-VI-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0xff
-; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
-; GISEL-VI-NEXT: v_min_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-VI-NEXT: v_min_i16_e32 v1, 0xff, v1
+; GISEL-VI-NEXT: v_min_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GISEL-VI-NEXT: v_or_b32_e32 v0, v1, v0
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX9-LABEL: vec_smax_smin:
@@ -729,14 +735,21 @@ define amdgpu_kernel void @vec_smax_smin_sgpr(ptr addrspace(1) %out, <2 x i16> i
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
; GISEL-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GISEL-VI-NEXT: v_mov_b32_e32 v0, 0xff
; GISEL-VI-NEXT: s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT: s_ashr_i32 s3, s2, 16
+; GISEL-VI-NEXT: s_lshr_b32 s3, s2, 16
+; GISEL-VI-NEXT: s_sext_i32_i16 s3, s3
+; GISEL-VI-NEXT: s_sext_i32_i16 s2, s2
+; GISEL-VI-NEXT: s_max_i32 s3, s3, 0
+; GISEL-VI-NEXT: s_max_i32 s2, s2, 0
+; GISEL-VI-NEXT: s_sext_i32_i16 s3, s3
; GISEL-VI-NEXT: s_sext_i32_i16 s2, s2
-; GISEL-VI-NEXT: v_med3_i32 v1, s2, 0, v0
-; GISEL-VI-NEXT: v_med3_i32 v0, s3, 0, v0
-; GISEL-VI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GISEL-VI-NEXT: v_or_b32_sdwa v2, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GISEL-VI-NEXT: s_min_i32 s3, s3, 0xff
+; GISEL-VI-NEXT: s_min_i32 s2, s2, 0xff
+; GISEL-VI-NEXT: s_and_b32 s3, 0xffff, s3
+; GISEL-VI-NEXT: s_and_b32 s2, 0xffff, s2
+; GISEL-VI-NEXT: s_lshl_b32 s3, s3, 16
+; GISEL-VI-NEXT: s_or_b32 s2, s2, s3
+; GISEL-VI-NEXT: v_mov_b32_e32 v2, s2
; GISEL-VI-NEXT: v_mov_b32_e32 v0, s0
; GISEL-VI-NEXT: v_mov_b32_e32 v1, s1
; GISEL-VI-NEXT: flat_store_dword v[0:1], v2
@@ -863,12 +876,12 @@ define <2 x i16> @vec_smin_smax(<2 x i16> %src) {
; GISEL-VI: ; %bb.0:
; GISEL-VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GISEL-VI-NEXT: v_mov_b32_e32 v1, 0xff
-; GISEL-VI-NEXT: v_min_i16_sdwa v1, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GISEL-VI-NEXT: v_min_i16_e32 v0, 0xff, v0
+; GISEL-VI-NEXT: v_min_i16_e32 v2, 0xff, v0
+; GISEL-VI-NEXT: v_min_i16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GISEL-VI-NEXT: v_max_i16_e32 v1, 0, v2
; GISEL-VI-NEXT: v_mov_b32_e32 v2, 0
-; GISEL-VI-NEXT: v_max_i16_e32 v0, 0, v0
-; GISEL-VI-NEXT: v_max_i16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GISEL-VI-NEXT: v_or_b32_e32 v0, v0, v1
+; GISEL-VI-NEXT: v_max_i16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GISEL-VI-NEXT: v_or_b32_e32 v0, v1, v0
; GISEL-VI-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-GFX9-LABEL: vec_smin_smax:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
index c10c8ce2946d5..0c90f196b9be3 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-and.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
@@ -1175,20 +1175,46 @@ entry:
}
define i16 @test_vector_reduce_and_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_and_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_and_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_and_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_and_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_and_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_and_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_and_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_and_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_and_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_and_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
index bf1e2583a7f4c..97668d8cde573 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-or.ll
@@ -998,21 +998,47 @@ entry:
}
define i16 @test_vector_reduce_or_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_or_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_or_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_or_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_or_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_or_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_or_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_or_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
index 1e9ba2f18b6ae..e7e4cc790e026 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-umin.ll
@@ -4,18 +4,18 @@
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_umin_v2i8(<2 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_umin_v2i8:
diff --git a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
index 57e884d85d437..210ce0195aeec 100644
--- a/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector-reduce-xor.ll
@@ -1,21 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu7.00 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu8.01 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu9.42 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu10.10 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu11.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
-; RUN: llc -global-isel=1 -global-isel-abort=2 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
+; RUN: llc -global-isel=1 -mtriple=amdgpu12.00 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
; FIXME: codegen regression, related to:
; - looking through sgpr to vgpr copy
@@ -1147,21 +1147,47 @@ entry:
}
define i16 @test_vector_reduce_xor_v4i16(<4 x i16> %v) {
-; GFX7-LABEL: test_vector_reduce_xor_v4i16:
-; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX7-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX7-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX7-NEXT: s_setpc_b64 s[30:31]
+; GFX7-SDAG-LABEL: test_vector_reduce_xor_v4i16:
+; GFX7-SDAG: ; %bb.0: ; %entry
+; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
-; GFX8-LABEL: test_vector_reduce_xor_v4i16:
-; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX8-NEXT: v_xor_b32_e32 v0, v0, v1
-; GFX8-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX8-NEXT: s_setpc_b64 s[30:31]
+; GFX7-GISEL-LABEL: test_vector_reduce_xor_v4i16:
+; GFX7-GISEL: ; %bb.0: ; %entry
+; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v3, 16, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX7-GISEL-NEXT: v_or_b32_e32 v0, v2, v0
+; GFX7-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX7-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-GISEL-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
+; GFX7-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-SDAG-LABEL: test_vector_reduce_xor_v4i16:
+; GFX8-SDAG: ; %bb.0: ; %entry
+; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-SDAG-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-SDAG-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX8-GISEL-LABEL: test_vector_reduce_xor_v4i16:
+; GFX8-GISEL: ; %bb.0: ; %entry
+; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX8-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX8-GISEL-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX8-GISEL-NEXT: v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-GISEL-NEXT: v_xor_b32_e32 v0, v0, v1
+; GFX8-GISEL-NEXT: v_xor_b32_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_xor_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
More information about the llvm-branch-commits
mailing list